From a91de68d4fd288c437530cb6aff6bc93d04439ad Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 19 Jan 2026 10:29:51 +0100 Subject: [PATCH 01/16] OAK-12010 Simplified index management (without optimizer) --- .../oak/plugins/index/IndexName.java | 17 + .../oak/plugins/index/IndexUpdate.java | 21 + .../oak/plugins/index/diff/DiffIndex.java | 246 ++++++ .../plugins/index/diff/DiffIndexMerger.java | 801 ++++++++++++++++++ .../plugins/index/diff/JsonNodeBuilder.java | 279 ++++++ .../index/diff/RootIndexesListService.java | 112 +++ .../oak/plugins/index/diff/DiffIndexTest.java | 307 +++++++ .../index/diff/JsonNodeBuilderTest.java | 226 +++++ .../oak/plugins/index/diff/MergeTest.java | 191 +++++ .../oak/plugins/index/diff/indexes.json | 187 ++++ .../index/search/spi/query/FulltextIndex.java | 2 + .../index/search/spi/query/IndexNameTest.java | 53 ++ 12 files changed, 2442 insertions(+) create mode 100644 oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java create mode 100644 oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java create mode 100644 oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java create mode 100644 oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/RootIndexesListService.java create mode 100644 oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java create mode 100644 oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java create mode 100644 oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java create mode 100644 oak-core/src/test/resources/org/apache/jackrabbit/oak/plugins/index/diff/indexes.json diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java index 3597079d28b..7d8313c8e22 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java @@ -234,6 +234,23 @@ public static Collection filterReplacedIndexes(Collection indexP return result; } + public static Collection filterNewestIndexes(Collection indexPaths, NodeState rootState) { + HashMap latestVersions = new HashMap<>(); + for (String p : indexPaths) { + IndexName indexName = IndexName.parse(p); + IndexName stored = latestVersions.get(indexName.baseName); + if (stored == null || stored.compareTo(indexName) < 0) { + // no old version, or old version is smaller: use + latestVersions.put(indexName.baseName, indexName); + } + } + ArrayList result = new ArrayList<>(latestVersions.size()); + for (IndexName n : latestVersions.values()) { + result.add(n.nodeName); + } + return result; + } + public String nextCustomizedName() { return baseName + "-" + productVersion + "-custom-" + (customerVersion + 1); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java index e33bfe9eff7..058d8f8b162 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexUpdate.java @@ -48,6 +48,8 @@ import org.apache.jackrabbit.oak.commons.collections.SetUtils; import org.apache.jackrabbit.oak.plugins.index.IndexCommitCallback.IndexProgress; import org.apache.jackrabbit.oak.plugins.index.NodeTraversalCallback.PathSource; +import org.apache.jackrabbit.oak.plugins.index.diff.DiffIndex; +import org.apache.jackrabbit.oak.plugins.index.diff.DiffIndexMerger; import org.apache.jackrabbit.oak.plugins.index.progress.IndexingProgressReporter; import org.apache.jackrabbit.oak.plugins.index.progress.NodeCountEstimator; import org.apache.jackrabbit.oak.plugins.index.progress.TraversalRateEstimator; @@ -60,6 +62,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; +import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder; import org.apache.jackrabbit.util.ISO8601; import org.jetbrains.annotations.NotNull; @@ -108,6 +111,8 @@ public class IndexUpdate implements Editor, PathSource { } } + private final NodeStore store; + private final IndexUpdateRootState rootState; private final NodeBuilder builder; @@ -150,6 +155,16 @@ public IndexUpdate( NodeState root, NodeBuilder builder, IndexUpdateCallback updateCallback, NodeTraversalCallback traversalCallback, CommitInfo commitInfo, CorruptIndexHandler corruptIndexHandler) { + this(provider, async, root, builder, updateCallback, traversalCallback, commitInfo, corruptIndexHandler, null); + } + + public IndexUpdate( + IndexEditorProvider provider, String async, + NodeState root, NodeBuilder builder, + IndexUpdateCallback updateCallback, NodeTraversalCallback traversalCallback, + CommitInfo commitInfo, CorruptIndexHandler corruptIndexHandler, + @Nullable NodeStore store) { + this.store = store; this.parent = null; this.name = null; this.path = "/"; @@ -158,6 +173,7 @@ public IndexUpdate( } private IndexUpdate(IndexUpdate parent, String name) { + this.store = parent.store; this.parent = requireNonNull(parent); this.name = name; this.rootState = parent.rootState; @@ -279,6 +295,11 @@ private static boolean hasAnyHiddenNodes(NodeBuilder builder) { } private void collectIndexEditors(NodeBuilder definitions, NodeState before) throws CommitFailedException { + if (definitions.hasChildNode(DiffIndexMerger.DIFF_INDEX) + && "disabled".equals(definitions.child(DiffIndexMerger.DIFF_INDEX).getString("type")) + && rootState.async == null) { + DiffIndex.applyDiffIndexChanges(store, definitions); + } for (String name : definitions.getChildNodeNames()) { NodeBuilder definition = definitions.getChildNode(name); if (isIncluded(rootState.async, definition)) { diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java new file mode 100644 index 00000000000..825a23c38ea --- /dev/null +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexName; +import org.apache.jackrabbit.oak.plugins.tree.TreeConstants; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Processing of diff indexes, that is nodes under "/oak:index/diff.index". A + * diff index contains differences to existing indexes, and possibly new + * (custom) indexes in the form of JSON. These changes can then be merged + * (applied) to the index definitions. This allows to simplify index management, + * because it allows to modify (add, update) indexes in a simple way. + */ +public class DiffIndex { + + private static final Logger LOG = LoggerFactory.getLogger(DiffIndex.class); + + /** + * Apply changes to the index definitions. That means merge the index diff with + * the existing indexes, creating new index versions. It might also mean to + * remove old (merged) indexes if the diff no longer contains them. + * + * @param store the node store + * @param indexDefinitions the /oak:index node + */ + public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefinitions) { + JsonObject newImageLuceneDefinitions = null; + for (String diffIndex : new String[] { DiffIndexMerger.DIFF_INDEX, DiffIndexMerger.DIFF_INDEX_OPTIMIZER }) { + if (!indexDefinitions.hasChildNode(diffIndex)) { + continue; + } + NodeBuilder diffIndexDefinition = indexDefinitions.child(diffIndex); + NodeBuilder diffJson = diffIndexDefinition.getChildNode("diff.json"); + if (!diffJson.exists()) { + continue; + } + NodeBuilder jcrContent = diffJson.getChildNode("jcr:content"); + if (!jcrContent.exists()) { + continue; + } + PropertyState lastMod = jcrContent.getProperty("jcr:lastModified"); + if (lastMod == null) { + continue; + } + String modified = lastMod.getValue(Type.DATE); + PropertyState lastProcessed = jcrContent.getProperty(":lastProcessed"); + if (lastProcessed != null) { + if (modified.equals(lastProcessed.getValue(Type.STRING))) { + // already processed + continue; + } + } + // store now, so a change is only processed once + jcrContent.setProperty(":lastProcessed", modified); + PropertyState jcrData = jcrContent.getProperty("jcr:data"); + String diff = tryReadString(jcrData); + if (diff == null) { + continue; + } + try { + JsonObject diffObj = JsonObject.fromJson("{\"diff\": " + diff + "}", true); + diffIndexDefinition.removeProperty("error"); + if (newImageLuceneDefinitions == null) { + newImageLuceneDefinitions = new JsonObject(); + } + newImageLuceneDefinitions.getChildren().put("/oak:index/" + diffIndex, diffObj); + } catch (Exception e) { + String message = "Error parsing diff.index"; + LOG.warn(message + ": {}", e.getMessage(), e); + diffIndexDefinition.setProperty("error", message + ": " + e.getMessage()); + } + } + if (newImageLuceneDefinitions == null) { + // not a valid diff index, or already processed + return; + } + LOG.info("Processing a new diff.index with node store {}", store); + JsonObject repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(indexDefinitions); + LOG.debug("Index list {}", repositoryDefinitions.toString()); + try { + DiffIndexMerger.merge(newImageLuceneDefinitions, repositoryDefinitions, store); + for (String indexPath : newImageLuceneDefinitions.getChildren().keySet()) { + if (indexPath.startsWith("/oak:index/" + DiffIndexMerger.DIFF_INDEX)) { + continue; + } + JsonObject newDef = newImageLuceneDefinitions.getChildren().get(indexPath); + String indexName = PathUtils.getName(indexPath); + JsonNodeBuilder.addOrReplace(indexDefinitions, store, indexName, IndexConstants.INDEX_DEFINITIONS_NODE_TYPE, newDef.toString()); + updateNodetypeIndexForPath(indexDefinitions, indexName, true); + disableOrRemoveOldVersions(indexDefinitions, indexPath, indexName); + } + removeDisabledMergedIndexes(indexDefinitions); + sortIndexes(indexDefinitions); + } catch (Exception e) { + LOG.warn("Error merging diff.index: {}", e.getMessage(), e); + NodeBuilder diffIndexDefinition = indexDefinitions.child(DiffIndexMerger.DIFF_INDEX); + diffIndexDefinition.setProperty("error", e.getMessage()); + } + } + + /** + * Try to read a text from the (binary) jcr:data property. Edge cases such as + * "property does not exist" and IO exceptions (blob not found) do not throw an + * exception (IO exceptions are logged). + * + * @param jcrData the "jcr:data" property + * @return the string, or null if reading fails + */ + public static String tryReadString(PropertyState jcrData) { + if (jcrData == null) { + return null; + } + InputStream in = jcrData.getValue(Type.BINARY).getNewStream(); + try { + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + LOG.warn("Can not read jcr:data", e); + return null; + } + } + + private static void sortIndexes(NodeBuilder builder) { + ArrayList list = new ArrayList<>(); + for (String child : builder.getChildNodeNames()) { + list.add(child); + } + list.sort(Comparator.naturalOrder()); + builder.setProperty(TreeConstants.OAK_CHILD_ORDER, list, Type.NAMES); + } + + private static void removeDisabledMergedIndexes(NodeBuilder definitions) { + ArrayList toRemove = new ArrayList<>(); + for (String child : definitions.getChildNodeNames()) { + if (!definitions.getChildNode(child).hasProperty("mergeChecksum")) { + continue; + } + if ("disabled".equals(definitions.getChildNode(child).getString("type"))) { + toRemove.add(child); + } + } + for (String r : toRemove) { + LOG.info("Removing disabled index {}", r); + definitions.child(r).remove(); + updateNodetypeIndexForPath(definitions, r, false); + } + } + + /** + * Try to remove or disable old version of merged indexes, if there are any. + * + * @param definitions the builder for /oak:index + * @param indexPath the path + * @param keep which index name (which version) to retain + */ + private static void disableOrRemoveOldVersions(NodeBuilder definitions, String indexPath, String keep) { + String indexName = indexPath; + if (indexPath.startsWith("/oak:index/")) { + indexName = indexPath.substring("/oak:index/".length()); + } + String baseName = IndexName.parse(indexName).getBaseName(); + ArrayList toRemove = new ArrayList<>(); + for (String child : definitions.getChildNodeNames()) { + if (child.equals(keep) || child.indexOf("-custom-") < 0) { + // the one to keep, or not a customized or custom index + continue; + } + String childBaseName = IndexName.parse(child).getBaseName(); + if (baseName.equals(childBaseName)) { + if (indexName.equals(child)) { + if (!"disabled".equals(definitions.getChildNode(indexName).getString("type"))) { + continue; + } + } + toRemove.add(child); + } + } + for (String r : toRemove) { + LOG.info("Removing old index " + r); + definitions.child(r).remove(); + updateNodetypeIndexForPath(definitions, r, false); + } + } + + private static void updateNodetypeIndexForPath(NodeBuilder indexDefinitions, + String indexName, boolean add) { + LOG.info("nodetype index update add={} name={}", add, indexName); + if (!indexDefinitions.hasChildNode("nodetype")) { + return; + } + NodeBuilder nodetypeIndex = indexDefinitions.getChildNode("nodetype"); + NodeBuilder indexContent = nodetypeIndex.child(":index"); + String key = URLEncoder.encode("oak:QueryIndexDefinition", StandardCharsets.UTF_8); + String path = "/oak:index/" + indexName; + if (add) { + // insert entry + NodeBuilder builder = indexContent.child(key); + for (String name : PathUtils.elements(path)) { + builder = builder.child(name); + } + LOG.info("nodetype index match"); + builder.setProperty("match", true); + } else { + // remove entry (for deleted indexes) + NodeBuilder builder = indexContent.getChildNode(key); + for (String name : PathUtils.elements(path)) { + builder = builder.getChildNode(name); + } + if (builder.exists()) { + LOG.info("nodetype index remove"); + builder.removeProperty("match"); + } + } + } + +} diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java new file mode 100644 index 00000000000..b5c45279769 --- /dev/null +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java @@ -0,0 +1,801 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.jackrabbit.oak.commons.StringUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.commons.json.JsopBuilder; +import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; +import org.apache.jackrabbit.oak.json.Base64BlobSerializer; +import org.apache.jackrabbit.oak.json.JsonSerializer; +import org.apache.jackrabbit.oak.plugins.index.IndexName; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Index definition merge utility that uses the "diff" mode. + */ +public class DiffIndexMerger { + + final static Logger LOG = LoggerFactory.getLogger(DiffIndexMerger.class); + + public final static String DIFF_INDEX = "diff.index"; + public final static String DIFF_INDEX_OPTIMIZER = "diff.index.optimizer"; + + private final static String MERGE_INFO = "This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html"; + + // the list of unsupported included paths, e.g. "/apps,/libs" + // by default all paths are supported + private final static String[] UNSUPPORTED_INCLUDED_PATHS = System.getProperty("oak.diffIndex.unsupportedPaths", "").split(","); + + // in case a custom index is removed, whether a dummy index is created + private final static boolean DELETE_CREATES_DUMMY = Boolean.getBoolean("oak.diffIndex.deleteCreatesDummy"); + + // in case a customization was removed, create a copy of the OOTB index + private final static boolean DELETE_COPIES_OOTB = Boolean.getBoolean("oak.diffIndex.deleteCopiesOOTB"); + + /** + * If there is a diff index, that is an index with prefix "diff.", then try to merge it. + * + * @param newImageLuceneDefinitions + * the new indexes + * (input and output) + * @param repositoryDefinitions + * the indexes in the writable repository + * (input) + * @param repositoryNodeStore + */ + public static void merge(JsonObject newImageLuceneDefinitions, JsonObject repositoryDefinitions, NodeStore repositoryNodeStore) { + // combine all definitions into one object + JsonObject combined = new JsonObject(); + + // index definitions in the repository + combined.getChildren().putAll(repositoryDefinitions.getChildren()); + + // read the diff.index.optimizer explicitly, + // because it's a not a regular index definition, + // and so in the repositoryDefinitions + if (repositoryNodeStore != null) { + Map diffInRepo = readDiffIndex(repositoryNodeStore, DIFF_INDEX_OPTIMIZER); + combined.getChildren().putAll(diffInRepo); + } + + // overwrite with the provided definitions (if any) + combined.getChildren().putAll(newImageLuceneDefinitions.getChildren()); + + // check if there "diff.index" or "diff.index.optimizer" + boolean found = combined.getChildren().containsKey("/oak:index/" + DIFF_INDEX) + || combined.getChildren().containsKey("/oak:index/" + DIFF_INDEX_OPTIMIZER); + if (!found) { + // early exit, so that the risk of merging the PR + // is very small for customers that do not use this + LOG.debug("No 'diff.index' definition"); + return; + } + mergeDiff(newImageLuceneDefinitions, combined); + } + + /** + * If there is a diff index (hardcoded node "/oak:index/diff.index" or + * "/oak:index/diff.index.optimizer"), then iterate over all entries and create new + * (merged) versions if needed. + * + * @param newImageLuceneDefinitions + * the new Lucene definitions + * (input + output) + * @param combined + * the definitions in the repository, + * including the one in the customer repo and new ones + * (input) + * @return whether a new version of an index was added + */ + static boolean mergeDiff(JsonObject newImageLuceneDefinitions, JsonObject combined) { + // iterate again, this time process + + // collect the diff index(es) + HashMap toProcess = new HashMap<>(); + tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX, toProcess); + tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX_OPTIMIZER, toProcess); + // if the diff index exists, but doesn't contain some of the previous indexes + // (indexes with mergeInfo), then we need to disable those (using /dummy includedPath) + extractExistingMergedIndexes(combined, toProcess); + if (toProcess.isEmpty()) { + LOG.debug("No diff index definitions found."); + return false; + } + boolean hasChanges = false; + for (Entry e : toProcess.entrySet()) { + String key = e.getKey(); + JsonObject value = e.getValue(); + if (key.startsWith("/oak:index/")) { + LOG.warn("The key should contains just the index name, without the '/oak:index' prefix for key {}", key); + key = key.substring("/oak:index/".length()); + } + LOG.debug("Processing {}", key); + hasChanges |= processMerge(key, value, newImageLuceneDefinitions, combined); + } + return hasChanges; + } + + /** + * Extract a "diff.index" from the set of index definitions (if found), and if + * found, store the nested entries in the target map, merging them with previous + * entries if found. + * + * The diff.index may either have a file (a "jcr:content" child node with a + * "jcr:data" property), or a "diff" JSON object. For customers (in the git + * repository), the file is much easier to construct, but when running the + * indexing job, the nested JSON is much easier. + * + * @param indexDefs the set of index definitions (may be empty) + * @param name the name of the diff.index (either diff.index or + * diff.index.optimizer) + * @param target the target map of diff.index definitions + * @return the error message trying to parse the JSON file, or null + */ + static String tryExtractDiffIndex(JsonObject indexDefs, String name, HashMap target) { + JsonObject diffIndex = indexDefs.getChildren().get(name); + if (diffIndex == null) { + return null; + } + // extract either the file, or the nested json + JsonObject file = diffIndex.getChildren().get("diff.json"); + JsonObject diff; + if (file != null) { + // file + JsonObject jcrContent = file.getChildren().get("jcr:content"); + if (jcrContent == null) { + String message = "jcr:content child node is missing in diff.json"; + LOG.warn(message); + return message; + } + String jcrData = JsonNodeBuilder.oakStringValue(jcrContent, "jcr:data"); + try { + diff = JsonObject.fromJson(jcrData, true); + } catch (Exception e) { + LOG.warn("Illegal Json, ignoring: {}", jcrData, e); + String message = "Illegal Json, ignoring: " + e.getMessage(); + return message; + } + } else { + // nested json + diff = diffIndex.getChildren().get("diff"); + } + // store, if not empty + if (diff != null) { + for (Entry e : diff.getChildren().entrySet()) { + String key = e.getKey(); + target.put(key, mergeDiffs(target.get(key), e.getValue())); + } + } + return null; + } + + /** + * Extract the indexes with a "mergeInfo" property and store them in the target + * object. This is needed so that indexes that were removed from the index.diff + * are detected (a new version is needed in this case with includedPaths + * "/dummy"). + * + * @param indexDefs the index definitions in the repository + * @param target the target map of "diff.index" definitions. for each entry + * found, an empty object is added + */ + private static void extractExistingMergedIndexes(JsonObject indexDefs, HashMap target) { + for (Entry e : indexDefs.getChildren().entrySet()) { + String key = e.getKey(); + JsonObject value = e.getValue(); + if (key.indexOf("-custom-") < 0 || !value.getProperties().containsKey("mergeInfo")) { + continue; + } + String baseName = IndexName.parse(key.substring("/oak:index/".length())).getBaseName(); + if (!target.containsKey(baseName)) { + // if there is no entry yet for this key, + // add a new empty object + target.put(baseName, new JsonObject()); + } + } + } + + /** + * Merge diff from "diff.index" and "diff.index.optimizer". + * The customer can define a diff (stored in "diff.index") + * and someone else (or the optimizer) can define one (stored in "diff.index.optimizer"). + * + * @param a the first diff + * @param b the second diff (overwrites entries in a) + * @return the merged entry + */ + private static JsonObject mergeDiffs(JsonObject a, JsonObject b) { + if (a == null) { + return b; + } else if (b == null) { + return a; + } + JsonObject result = JsonObject.fromJson(a.toString(), true); + result.getProperties().putAll(b.getProperties()); + HashSet both = new HashSet<>(a.getChildren().keySet()); + both.addAll(b.getChildren().keySet()); + for (String k : both) { + result.getChildren().put(k, mergeDiffs(a.getChildren().get(k), b.getChildren().get(k))); + } + return result; + } + + /** + * Merge using the diff definition. + * + * If the latest customized index already matches, then + * newImageLuceneDefinitions will remain as is. Otherwise, a new customized + * index is added, with a "mergeInfo" property. + * + * Existing properties are never changed; only new properties/children are + * added. + * + * @param indexName the name, eg. "damAssetLucene" + * @param indexDiff the diff with the new properties + * @param newImageLuceneDefinitions the new Lucene definitions (input + output) + * @param combined the definitions in the repository, including + * the one in the customer repo and new ones + * (input) + * @return whether a new version of an index was added + */ + public static boolean processMerge(String indexName, JsonObject indexDiff, JsonObject newImageLuceneDefinitions, JsonObject combined) { + // extract the latest product index (eg. damAssetLucene-12) + // and customized index (eg. damAssetLucene-12-custom-3) - if any + IndexName latestProduct = null; + String latestProductKey = null; + IndexName latestCustomized = null; + String latestCustomizedKey = null; + String prefix = "/oak:index/"; + for (String key : combined.getChildren().keySet()) { + IndexName name = IndexName.parse(key.substring(prefix.length())); + if (!name.isVersioned()) { + LOG.debug("Ignoring unversioned index {}", name); + continue; + } + if (!name.getBaseName().equals(indexName)) { + continue; + } + boolean isCustom = key.indexOf("-custom-") >= 0; + if (isCustom) { + if (latestCustomized == null || + name.compareTo(latestCustomized) > 0) { + latestCustomized = name; + latestCustomizedKey = key; + } + } else { + if (latestProduct == null || + name.compareTo(latestProduct) > 0) { + latestProduct = name; + latestProductKey = key; + } + } + } + LOG.debug("Latest product: {}", latestProductKey); + LOG.debug("Latest customized: {}", latestCustomizedKey); + if (latestProduct == null) { + if (indexName.indexOf('.') >= 0) { + // a fully custom index needs to contains a dot + LOG.debug("Fully custom index {}", indexName); + } else { + LOG.debug("No product version for {}", indexName); + return false; + } + } + JsonObject latestProductIndex = combined.getChildren().get(latestProductKey); + String[] includedPaths; + if (latestProductIndex == null) { + if (indexDiff.getProperties().isEmpty() && indexDiff.getChildren().isEmpty()) { + // there is no customization (any more), which means a dummy index may be needed + LOG.debug("No customization for {}", indexName); + } else { + includedPaths = JsonNodeBuilder.oakStringArrayValue(indexDiff, "includedPaths"); + if (includesUnsupportedPaths(includedPaths)) { + LOG.warn("New custom index {} is not supported because it contains an unsupported path ({})", + indexName, Arrays.toString(UNSUPPORTED_INCLUDED_PATHS)); + return false; + } + } + } else { + includedPaths = JsonNodeBuilder.oakStringArrayValue(latestProductIndex, "includedPaths"); + if (includesUnsupportedPaths(includedPaths)) { + LOG.warn("Customizing index {} is not supported because it contains an unsupported path ({})", + latestProductKey, Arrays.toString(UNSUPPORTED_INCLUDED_PATHS)); + return false; + } + } + + // merge + JsonObject merged = null; + if (indexDiff == null) { + // no diff definition: use to the OOTB index + if (latestCustomized == null) { + LOG.debug("Only a product index found, nothing to do"); + return false; + } + merged = latestProductIndex; + } else { + merged = processMerge(latestProductIndex, indexDiff); + } + + // compare to the latest version of the this index + JsonObject latestIndexVersion = new JsonObject(); + if (latestCustomized == null) { + latestIndexVersion = latestProductIndex; + } else { + latestIndexVersion = combined.getChildren().get(latestCustomizedKey); + } + JsonObject mergedDef = cleanedAndNormalized(switchToLucene(merged)); + // compute merge checksum for later, but do not yet add + String mergeChecksum = computeMergeChecksum(mergedDef); + // get the merge checksum before cleaning (cleaning removes it) - if available + String key; + if (latestIndexVersion == null) { + // new index + key = prefix + indexName + "-1-custom-1"; + } else { + String latestMergeChecksum = JsonNodeBuilder.oakStringValue(latestIndexVersion, "mergeChecksum"); + JsonObject latestDef = cleanedAndNormalized(switchToLucene(latestIndexVersion)); + if (isSameIgnorePropertyOrder(mergedDef, latestDef)) { + // normal case: no change + // (even if checksums do not match: checksums might be missing or manipulated) + LOG.debug("Latest index matches"); + if (latestMergeChecksum != null && !latestMergeChecksum.equals(mergeChecksum)) { + LOG.warn("Indexes do match, but checksums do not. Possibly checksum was changed: {} vs {}", latestMergeChecksum, mergeChecksum); + LOG.warn("latest: {}\nmerged: {}", latestDef, mergedDef); + } + return false; + } + if (latestMergeChecksum != null && latestMergeChecksum.equals(mergeChecksum)) { + // checksum matches, but data does not match + // could be eg. due to numbers formatting issues (-0.0 vs 0.0, 0.001 vs 1e-3) + // but unexpected because we do not normally have such cases + LOG.warn("Indexes do not match, but checksums match. Possible normalization issue."); + LOG.warn("Index: {}, latest: {}\nmerged: {}", indexName, latestDef, mergedDef); + // if checksums match, we consider it a match + return false; + } + LOG.info("Indexes do not match, with"); + LOG.info("Index: {}, latest: {}\nmerged: {}", indexName, latestDef, mergedDef); + // a new merged index definition + if (latestProduct == null) { + // fully custom index: increment version + key = prefix + indexName + + "-" + latestCustomized.getProductVersion() + + "-custom-" + (latestCustomized.getCustomerVersion() + 1); + } else { + // customized OOTB index: use the latest product as the base + key = prefix + indexName + + "-" + latestProduct.getProductVersion() + + "-custom-"; + if (latestCustomized != null) { + key += (latestCustomized.getCustomerVersion() + 1); + } else { + key += "1"; + } + } + } + merged.getProperties().put("mergeInfo", JsopBuilder.encode(MERGE_INFO)); + merged.getProperties().put("mergeChecksum", JsopBuilder.encode(mergeChecksum)); + merged.getProperties().put("merges", "[" + JsopBuilder.encode("/oak:index/" + indexName) + "]"); + merged.getProperties().remove("reindexCount"); + merged.getProperties().remove("reindex"); + if (!DELETE_COPIES_OOTB && indexDiff.toString().equals("{}")) { + merged.getProperties().put("type", "\"disabled\""); + merged.getProperties().put("mergeComment", "\"This index is superseeded and can be removed\""); + } + newImageLuceneDefinitions.getChildren().put(key, merged); + return true; + } + + /** + * Check whether the includedPaths covers unsupported paths, + * if there are any unsupported path (eg. "/apps" or "/libs"). + * In this case, simplified index management is not supported. + * + * @param includedPaths the includedPaths list + * @return true if any unsupported path is included + */ + public static boolean includesUnsupportedPaths(String[] includedPaths) { + if (UNSUPPORTED_INCLUDED_PATHS.length == 1 && "".equals(UNSUPPORTED_INCLUDED_PATHS[0])) { + // set to an empty string + return false; + } + if (includedPaths == null) { + // not set means all entries + return true; + } + for (String path : includedPaths) { + if ("/".equals(path)) { + // all + return true; + } + for (String unsupported : UNSUPPORTED_INCLUDED_PATHS) { + if (unsupported.isEmpty()) { + continue; + } + if (path.equals(unsupported) || path.startsWith(unsupported + "/")) { + // includedPaths matches, or starts with an unsupported path + return true; + } + } + } + return false; + } + + /** + * Compute the SHA-256 checksum of the JSON object. This is useful to detect + * that the JSON object was not "significantly" changed, even if stored + * somewhere and later read again. Insignificant changes include: rounding of + * floating point numbers, re-ordering properties, things like that. Without the + * checksum, we would risk creating a new version of a customized index each + * time the indexing job is run, even thought the customer didn't change + * anything. + * + * @param json the input + * @return the SHA-256 checksum + */ + private static String computeMergeChecksum(JsonObject json) { + byte[] bytes = json.toString().getBytes(StandardCharsets.UTF_8); + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + return StringUtils.convertBytesToHex(md.digest(bytes)); + } catch (NoSuchAlgorithmException e) { + // SHA-256 is guaranteed to be available in standard Java platforms + throw new RuntimeException("SHA-256 algorithm not available", e); + } + } + + /** + * Switch the index from type "elasticsearch" to "lucene", if needed. This will + * also replace all properties that have an "...@lucene" version. + * + * This is needed because we want to merge only the "lucene" version, to + * simplify the merging logic. (The switch to the "elasticsearch" version + * happens later). + * + * @param indexDef the index definition (is not changed by this method) + * @return the lucene version (a new JSON object) + */ + public static JsonObject switchToLucene(JsonObject indexDef) { + JsonObject obj = JsonObject.fromJson(indexDef.toString(), true); + String type = JsonNodeBuilder.oakStringValue(obj, "type"); + if (type == null || !"elasticsearch".equals(type) ) { + return obj; + } + switchToLuceneChildren(obj); + return obj; + } + + private static void switchToLuceneChildren(JsonObject indexDef) { + // clone the keys to avoid ConcurrentModificationException + for (String p : new ArrayList<>(indexDef.getProperties().keySet())) { + if (!p.endsWith("@lucene")) { + continue; + } + String v = indexDef.getProperties().remove(p); + indexDef.getProperties().put(p.substring(0, p.length() - "@lucene".length()), v); + } + for (String c : indexDef.getChildren().keySet()) { + JsonObject co = indexDef.getChildren().get(c); + switchToLuceneChildren(co); + } + } + + /** + * Convert the JSON object to a new object, where index definition + * properties that are unimportant for comparison are removed. + * Example of important properties are "reindex", "refresh", "seed" etc. + * The order of properties is not relevant (but the order of children is). + * + * @param obj the input (is not changed by the method) + * @return a new JSON object + */ + public static JsonObject cleanedAndNormalized(JsonObject obj) { + obj = JsonObject.fromJson(obj.toString(), true); + obj.getProperties().remove(":version"); + obj.getProperties().remove(":nameSeed"); + obj.getProperties().remove(":mappingVersion"); + obj.getProperties().remove("refresh"); + obj.getProperties().remove("reindexCount"); + obj.getProperties().remove("reindex"); + obj.getProperties().remove("seed"); + obj.getProperties().remove("merges"); + obj.getProperties().remove("mergeInfo"); + obj.getProperties().remove("mergeChecksum"); + for (String p : new ArrayList<>(obj.getProperties().keySet())) { + if (p.endsWith("@lucene")) { + obj.getProperties().remove(p); + } else if (p.endsWith("@elasticsearch")) { + obj.getProperties().remove(p); + } else { + // remove "str:", "nam:", etc if needed + String v = obj.getProperties().get(p); + String v2 = normalizeOakString(v); + if (!v2.equals(v)) { + obj.getProperties().put(p, v2); + } + } + } + removeUUIDs(obj); + for (Entry e : obj.getChildren().entrySet()) { + obj.getChildren().put(e.getKey(), cleanedAndNormalized(e.getValue())); + } + // re-build the properties in alphabetical order + // (sorting the child nodes would be incorrect however, as order is significant here) + TreeMap props = new TreeMap<>(obj.getProperties()); + obj.getProperties().clear(); + for (Entry e : props.entrySet()) { + obj.getProperties().put(e.getKey(), e.getValue()); + } + return obj; + } + + /** + * "Normalize" a JSON string value. Remove any "nam:" and "dat:" and "str:" + * prefix in the value, because customers won't use them normally. (We want the + * diff to be as simple as possible). + * + * @param value the value (including double quotes; eg. "str:value") + * @return the normalized value (including double quotes) + */ + private static String normalizeOakString(String value) { + if (value == null || !value.startsWith("\"")) { + // ignore numbers + return value; + } + value = JsopTokenizer.decodeQuoted(value); + if (value.startsWith("str:") || value.startsWith("nam:") || value.startsWith("dat:")) { + value = value.substring("str:".length()); + } + return JsopBuilder.encode(value); + } + + /** + * Remove all "jcr:uuid" properties (including those in children), because the + * values might conflict. (new uuids are added later when needed). + * + * @param obj the JSON object where uuids will be removed. + */ + private static void removeUUIDs(JsonObject obj) { + obj.getProperties().remove("jcr:uuid"); + for (JsonObject c : obj.getChildren().values()) { + removeUUIDs(c); + } + } + + /** + * Merge a product index with a diff. If the product index is null, then the + * diff needs to contain a complete custom index definition. + * + * @param productIndex the product index definition, or null if none + * @param diff the diff (from the diff.index definition) + * @return the index definition of the merged index + */ + public static JsonObject processMerge(JsonObject productIndex, JsonObject diff) { + JsonObject result; + if (productIndex == null) { + // fully custom index + result = new JsonObject(true); + } else { + result = JsonObject.fromJson(productIndex.toString(), true); + } + mergeInto("", diff, result); + addPrimaryType("", result); + return result; + } + + /** + * Add primary type properties where needed. For the top-level index definition, + * this is "oak:QueryIndexDefinition", and "nt:unstructured" elsewhere. + * + * @param path the path (so we can call the method recursively) + * @param json the JSON object (is changed if needed) + */ + private static void addPrimaryType(String path, JsonObject json) { + // all nodes need to have a node type; + // the index definition itself (at root level) is "oak:QueryIndexDefinition", + // and all other nodes are "nt:unstructured" + if (!json.getProperties().containsKey("jcr:primaryType")) { + // all nodes need to have a primary type, + // otherwise index import will fail + String nodeType; + if (path.isEmpty()) { + nodeType = "oak:QueryIndexDefinition"; + } else { + nodeType = "nt:unstructured"; + } + String nodeTypeValue = "nam:" + nodeType; + json.getProperties().put("jcr:primaryType", JsopBuilder.encode(nodeTypeValue)); + } + for (Entry e : json.getChildren().entrySet()) { + addPrimaryType(path + "/" + e.getKey(), e.getValue()); + } + } + + /** + * Merge a JSON diff into a target index definition. + * + * @param path the path + * @param diff the diff (what to merge) + * @param target where to merge into + */ + private static void mergeInto(String path, JsonObject diff, JsonObject target) { + for (String p : diff.getProperties().keySet()) { + if (path.isEmpty()) { + if ("jcr:primaryType".equals(p)) { + continue; + } + } + if (target.getProperties().containsKey(p)) { + // we do not currently allow to overwrite most existing properties + if (p.equals("boost")) { + // allow overwriting the boost value + LOG.info("Overwrite property {} value at {}", p, path); + target.getProperties().put(p, diff.getProperties().get(p)); + } else { + LOG.warn("Ignoring existing property {} at {}", p, path); + } + } else { + target.getProperties().put(p, diff.getProperties().get(p)); + } + } + for (String c : diff.getChildren().keySet()) { + String targetChildName = c; + if (!target.getChildren().containsKey(c)) { + if (path.endsWith("/properties")) { + // search for a property with the same "name" value + String propertyName = diff.getChildren().get(c).getProperties().get("name"); + if (propertyName != null) { + propertyName = JsonNodeBuilder.oakStringValue(propertyName); + String c2 = getChildWithKeyValuePair(target, "name", propertyName); + if (c2 != null) { + targetChildName = c2; + } + } + // search for a property with the same "function" value + String function = diff.getChildren().get(c).getProperties().get("function"); + if (function != null) { + function = JsonNodeBuilder.oakStringValue(function); + String c2 = getChildWithKeyValuePair(target, "function", function); + if (c2 != null) { + targetChildName = c2; + } + } + } + if (targetChildName.equals(c)) { + // only create the child (properties are added below) + target.getChildren().put(c, new JsonObject()); + } + } + mergeInto(path + "/" + targetChildName, diff.getChildren().get(c), target.getChildren().get(targetChildName)); + } + if (target.getProperties().isEmpty() && target.getChildren().isEmpty()) { + if (DELETE_CREATES_DUMMY) { + // dummy index + target.getProperties().put("async", "\"async\""); + target.getProperties().put("includedPaths", "\"/dummy\""); + target.getProperties().put("queryPaths", "\"/dummy\""); + target.getProperties().put("type", "\"lucene\""); + JsopBuilder buff = new JsopBuilder(); + buff.object(). + key("properties").object(). + key("dummy").object(). + key("name").value("dummy"). + key("propertyIndex").value(true). + endObject(). + endObject(). + endObject(); + JsonObject indexRules = JsonObject.fromJson(buff.toString(), true); + target.getChildren().put("indexRules", indexRules); + } else { + target.getProperties().put("type", "\"disabled\""); + } + } + } + + public static String getChildWithKeyValuePair(JsonObject obj, String key, String value) { + for(Entry c : obj.getChildren().entrySet()) { + String v2 = c.getValue().getProperties().get(key); + if (v2 == null) { + continue; + } + v2 = JsonNodeBuilder.oakStringValue(v2); + if (value.equals(v2)) { + return c.getKey(); + } + } + return null; + } + + /** + * Compare two JSON object, ignoring the order of properties. (The order of + * children is however significant). + * + * This is done in addition to the checksum comparison, because the in theory + * the customer might change the checksum (it is not read-only as read-only + * values are not supported). We do not rely on the comparison, but if comparison + * and checksum comparison do not match, we log a warning. + * + * @param a the first object + * @param b the second object + * @return true if the keys and values are equal + */ + public static boolean isSameIgnorePropertyOrder(JsonObject a, JsonObject b) { + if (!a.getChildren().keySet().equals(b.getChildren().keySet())) { + LOG.debug("Child (order) difference: {} vs {}", + a.getChildren().keySet(), b.getChildren().keySet()); + return false; + } + for (String k : a.getChildren().keySet()) { + if (!isSameIgnorePropertyOrder( + a.getChildren().get(k), b.getChildren().get(k))) { + return false; + } + } + TreeMap pa = new TreeMap<>(a.getProperties()); + TreeMap pb = new TreeMap<>(b.getProperties()); + if (!pa.toString().equals(pb.toString())) { + LOG.debug("Property value difference: {} vs {}", pa.toString(), pb.toString()); + } + return pa.toString().equals(pb.toString()); + } + + /** + * Read a diff.index from the repository, if it exists. + * This is needed because the build-transform job doesn't have this + * data: it is only available in the writeable repository. + * + * @param repositoryNodeStore the node store + * @return a map, possibly with a single entry with this key + */ + static Map readDiffIndex(NodeStore repositoryNodeStore, String name) { + HashMap map = new HashMap<>(); + NodeState root = repositoryNodeStore.getRoot(); + String indexPath = "/oak:index/" + name; + NodeState idxState = NodeStateUtils.getNode(root, indexPath); + LOG.debug("Searching index {}: found={}", indexPath, idxState.exists()); + if (!idxState.exists()) { + return map; + } + JsopBuilder builder = new JsopBuilder(); + String filter = "{\"properties\":[\"*\", \"-:childOrder\"],\"nodes\":[\"*\", \"-:*\"]}"; + JsonSerializer serializer = new JsonSerializer(builder, filter, new Base64BlobSerializer()); + serializer.serialize(idxState); + JsonObject jsonObj = JsonObject.fromJson(builder.toString(), true); + jsonObj = cleanedAndNormalized(jsonObj); + LOG.debug("Found {}", jsonObj.toString()); + map.put(indexPath, jsonObj); + return map; + } + +} diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java new file mode 100644 index 00000000000..2f290748a97 --- /dev/null +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.util.Map.Entry; +import java.util.TreeSet; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.UUID; + +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.commons.json.JsopReader; +import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.plugins.tree.TreeConstants; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A utility class to persist a configuration that is in the form of JSON into + * the node store. + * + * This is used to persist a small set of configuration nodes, eg. index + * definitions, using a simple JSON format. + * + * The node type does not need to be set on a per-node basis. Where it is + * missing, the provided node type is used (e.g. "nt:unstructured") + * + * A "jcr:uuid" is automatically added for nodes of type "nt:resource". + * + * String, string arrays, boolean, blob, long, and double values are supported. + * Values that start with ":blobId:...base64..." are stored as binaries. "str:", + * "nam:" and "dat:" prefixes are removed. + * + * "null" entries are not supported. + */ +public class JsonNodeBuilder { + + private static final Logger LOG = LoggerFactory.getLogger(JsonNodeBuilder.class); + + /** + * Add a replace a node, including all child nodes, in the node store. + * + * @param nodeStore the target node store + * @param targetPath the target path where the node(s) is/are replaced + * @param nodeType the node type of the new node (eg. "nt:unstructured") + * @param jsonString the json string with the node data + * @throws CommitFailedException if storing the nodes failed + * @throws IOException if storing a blob failed + */ + public static void addOrReplace(NodeBuilder builder, NodeStore nodeStore, String targetPath, String nodeType, String jsonString) throws CommitFailedException, IOException { + LOG.info("Storing {}: {}", targetPath, jsonString); + if (nodeType.indexOf("/") >= 0) { + throw new IllegalStateException("Illegal node type: " + nodeType); + } + JsonObject json = JsonObject.fromJson(jsonString, true); + for (String name : PathUtils.elements(targetPath)) { + NodeBuilder child = builder.child(name); + if (!child.hasProperty("jcr:primaryType")) { + child.setProperty("jcr:primaryType", nodeType, Type.NAME); + } + builder = child; + } + storeConfigNode(nodeStore, builder, nodeType, json); + } + + private static void storeConfigNode(NodeStore nodeStore, NodeBuilder builder, String nodeType, JsonObject json) throws IOException { + ArrayList childOrder = new ArrayList<>(); + for (Entry e : json.getChildren().entrySet()) { + String k = e.getKey(); + childOrder.add(k); + JsonObject v = e.getValue(); + storeConfigNode(nodeStore, builder.child(k), nodeType, v); + } + for (String child : builder.getChildNodeNames()) { + if (!json.getChildren().containsKey(child)) { + builder.child(child).remove(); + } + } + for (Entry e : json.getProperties().entrySet()) { + String k = e.getKey(); + String v = e.getValue(); + storeConfigProperty(nodeStore, builder, k, v); + } + if (!json.getProperties().containsKey("jcr:primaryType")) { + builder.setProperty("jcr:primaryType", nodeType, Type.NAME); + } + for (PropertyState prop : builder.getProperties()) { + if ("jcr:primaryType".equals(prop.getName())) { + continue; + } + if (!json.getProperties().containsKey(prop.getName())) { + builder.removeProperty(prop.getName()); + } + } + builder.setProperty(TreeConstants.OAK_CHILD_ORDER, childOrder, Type.NAMES); + if ("nt:resource".equals(JsonNodeBuilder.oakStringValue(json, "jcr:primaryType"))) { + if (!json.getProperties().containsKey("jcr:uuid")) { + String uuid = UUID.randomUUID().toString(); + builder.setProperty("jcr:uuid", uuid); + } + } + } + + private static void storeConfigProperty(NodeStore nodeStore, NodeBuilder builder, String propertyName, String value) throws IOException { + if (value.startsWith("\"")) { + // string or blob + value = JsopTokenizer.decodeQuoted(value); + if (value.startsWith(":blobId:")) { + String base64 = value.substring(":blobId:".length()); + byte[] bytes = Base64.getDecoder().decode(base64.getBytes(StandardCharsets.UTF_8)); + if (nodeStore == null) { + MemoryNodeStore mns = new MemoryNodeStore(); + Blob blob = mns.createBlob(new ByteArrayInputStream(bytes)); + builder.setProperty(propertyName, blob); + } else { + Blob blob = nodeStore.createBlob(new ByteArrayInputStream(bytes)); + builder.setProperty(propertyName, blob); + } + } else { + if (value.startsWith("str:") || value.startsWith("nam:") || value.startsWith("dat:")) { + value = value.substring("str:".length()); + } + if ("jcr:primaryType".equals(propertyName)) { + builder.setProperty(propertyName, value, Type.NAME); + } else { + builder.setProperty(propertyName, value); + } + } + } else if ("null".equals(value)) { + throw new IllegalArgumentException("Removing entries is not supported for property " + propertyName); + } else if ("true".equals(value)) { + builder.setProperty(propertyName, true); + } else if ("false".equals(value)) { + builder.setProperty(propertyName, false); + } else if (value.startsWith("[")) { + JsopTokenizer tokenizer = new JsopTokenizer(value); + ArrayList result = new ArrayList<>(); + tokenizer.matches('['); + if (!tokenizer.matches(']')) { + do { + if (!tokenizer.matches(JsopReader.STRING)) { + throw new IllegalArgumentException("Could not process string array " + value + " for property " + propertyName); + } + result.add(tokenizer.getToken()); + } while (tokenizer.matches(',')); + tokenizer.read(']'); + } + tokenizer.read(JsopReader.END); + builder.setProperty(propertyName, result, Type.STRINGS); + } else if (value.indexOf('.') >= 0 || value.toLowerCase().indexOf("e") >= 0) { + // double + try { + Double d = Double.parseDouble(value); + builder.setProperty(propertyName, d); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Could not parse double " + value + " for property " + propertyName); + } + } else if (value.startsWith("-") || (!value.isEmpty() && Character.isDigit(value.charAt(0)))) { + // long + try { + Long x = Long.parseLong(value); + builder.setProperty(propertyName, x); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Could not parse long " + value + " for property " + propertyName); + } + } else { + throw new IllegalArgumentException("Unsupported value " + value + " for property " + propertyName); + } + } + + public static String oakStringValue(JsonObject json, String propertyName) { + String value = json.getProperties().get(propertyName); + if (value == null) { + return null; + } + return oakStringValue(value); + } + + public static String oakStringValue(String value) { + if (!value.startsWith("\"")) { + // support numbers + return value; + } + value = JsopTokenizer.decodeQuoted(value); + if (value.startsWith(":blobId:")) { + value = value.substring(":blobId:".length()); + value = new String(Base64.getDecoder().decode(value.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); + } else if (value.startsWith("str:") || value.startsWith("nam:") || value.startsWith("dat:")) { + value = value.substring("str:".length()); + } + return value; + } + + /** + * Read an Oak string array. There are 3 cases: + * + * - the property doesn't exist: return null + * - the value is stored as string: return an array with one entry + * - the value is stored in an array: return the sorted list of value + * + * The value is sorted, because the order is insignificant in our case, + * and want ["a", "b"] = ["b", "a"] when comparing index definitions. + * + * @param json the JSON object + * @param propertyName the property to extract + * @return a string array or null + */ + public static String[] oakStringArrayValue(JsonObject json, String propertyName) { + String value = json.getProperties().get(propertyName); + if (value == null) { + return null; + } else if (value.startsWith("\"")) { + return new String[] { oakStringValue(value) }; + } else if (value.startsWith("[")) { + return getStringSet(value).toArray(new String[0]); + } else { + LOG.warn("Unsupported value type: {}", value); + return null; + } + } + + public static TreeSet getStringSet(String value) { + if (value == null) { + return null; + } + try { + JsopTokenizer tokenizer = new JsopTokenizer(value); + TreeSet result = new TreeSet<>(); + if (tokenizer.matches(JsopReader.STRING)) { + result.add(tokenizer.getToken()); + return result; + } + if (!tokenizer.matches('[')) { + return null; + } + if (!tokenizer.matches(']')) { + do { + if (!tokenizer.matches(JsopReader.STRING)) { + // not a string + return null; + } + result.add(tokenizer.getToken()); + } while (tokenizer.matches(',')); + tokenizer.read(']'); + } + tokenizer.read(JsopReader.END); + return result; + } catch (IllegalArgumentException e) { + LOG.warn("Unsupported value: {}", value); + return null; + } + } + +} diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/RootIndexesListService.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/RootIndexesListService.java new file mode 100644 index 00000000000..806278f1540 --- /dev/null +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/RootIndexesListService.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; + +import org.apache.felix.inventory.Format; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.commons.json.JsopBuilder; +import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; +import org.apache.jackrabbit.oak.json.Base64BlobSerializer; +import org.apache.jackrabbit.oak.json.JsonSerializer; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexPathService; +import org.apache.jackrabbit.oak.plugins.index.inventory.IndexDefinitionPrinter; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.jetbrains.annotations.Nullable; + +public class RootIndexesListService implements IndexPathService { + + private final NodeStore nodeStore; + + private RootIndexesListService(NodeStore nodeStore) { + this.nodeStore = nodeStore; + } + + public static JsonObject getRootIndexDefinitions(NodeBuilder definitions) { + JsopBuilder json = new JsopBuilder(); + String filter = "{\"properties\":[\"*\", \"-:childOrder\"],\"nodes\":[\"*\", \"-:*\"]}"; + json.object(); + for (String indexPath : definitions.getChildNodeNames()) { + NodeState node = definitions.child(indexPath).getNodeState(); + json.key("/oak:index/" + indexPath); + JsonSerializer s = new JsonSerializer(json, filter, new Base64BlobSerializer()); + s.serialize(node); + } + json.endObject(); + return JsonObject.fromJson(json.toString(), true); + } + + /** + * Get the index definitions at /oak:index from a node store. + * + * @param nodeStore the source node store (may not be null) + * @param typePattern the index types (may be null, meaning all) + * @return a JSON object with all index definitions + */ + public static JsonObject getRootIndexDefinitions(NodeStore nodeStore, @Nullable String typePattern) { + if (nodeStore == null) { + return new JsonObject(); + } + RootIndexesListService imageIndexPathService = new RootIndexesListService(nodeStore); + IndexDefinitionPrinter indexDefinitionPrinter = new IndexDefinitionPrinter(nodeStore, imageIndexPathService); + StringWriter writer = new StringWriter(); + PrintWriter printWriter = new PrintWriter(writer); + indexDefinitionPrinter.print(printWriter, Format.JSON, false); + printWriter.flush(); + writer.flush(); + String json = writer.toString(); + JsonObject result = JsonObject.fromJson(json, true); + if (typePattern != null) { + for (String c : new ArrayList<>(result.getChildren().keySet())) { + String type = result.getChildren().get(c).getProperties().get("type"); + if (type == null) { + continue; + } + type = JsopTokenizer.decodeQuoted(type); + if (type != null && !type.matches(typePattern)) { + result.getChildren().remove(c); + } + } + } + return result; + } + + @Override + public Iterable getIndexPaths() { + ArrayList list = new ArrayList<>(); + NodeState oakIndex = nodeStore.getRoot().getChildNode("oak:index"); + if (!oakIndex.exists()) { + return list; + } + for (ChildNodeEntry cn : oakIndex.getChildNodeEntries()) { + if (!IndexConstants.INDEX_DEFINITIONS_NODE_TYPE + .equals(cn.getNodeState().getName("jcr:primaryType"))) { + continue; + } + list.add("/oak:index/" + cn.getName()); + } + return list; + } + +} \ No newline at end of file diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java new file mode 100644 index 00000000000..888eeaf8983 --- /dev/null +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java @@ -0,0 +1,307 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.apache.jackrabbit.JcrConstants; +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate; +import org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider; +import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.reference.ReferenceEditorProvider; +import org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EditorHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.junit.Test; + +/** + * Tests for DiffIndex functionality. + */ +public class DiffIndexTest { + + @Test + public void listIndexes() { + NodeStore store = new MemoryNodeStore(INITIAL_CONTENT); + JsonObject indexDefs = RootIndexesListService.getRootIndexDefinitions(store, "property"); + // expect at least one index + assertFalse(indexDefs.getChildren().isEmpty()); + } + + @Test + public void tryReadStringNull() { + assertNull(DiffIndex.tryReadString(null)); + } + + @Test + public void tryReadStringValidContent() { + String content = "Hello, World!"; + PropertyState prop = BinaryPropertyState.binaryProperty("jcr:data", + content.getBytes(StandardCharsets.UTF_8)); + assertEquals(content, DiffIndex.tryReadString(prop)); + } + + @Test + public void tryReadStringEmpty() { + PropertyState prop = BinaryPropertyState.binaryProperty("jcr:data", new byte[0]); + assertEquals("", DiffIndex.tryReadString(prop)); + } + + @Test + public void tryReadStringJsonContent() { + String content = "{ \"key\": \"value\", \"array\": [1, 2, 3] }"; + PropertyState prop = BinaryPropertyState.binaryProperty("jcr:data", + content.getBytes(StandardCharsets.UTF_8)); + assertEquals(content, DiffIndex.tryReadString(prop)); + } + + @Test + public void tryReadStringIOException() throws IOException { + PropertyState prop = mock(PropertyState.class); + Blob blob = mock(Blob.class); + InputStream failingStream = new InputStream() { + @Override + public int read() throws IOException { + throw new IOException("Simulated read failure"); + } + @Override + public byte[] readAllBytes() throws IOException { + throw new IOException("Simulated read failure"); + } + }; + when(prop.getValue(Type.BINARY)).thenReturn(blob); + when(blob.getNewStream()).thenReturn(failingStream); + + // Should return null (not throw exception) + assertNull(DiffIndex.tryReadString(prop)); + } + + @Test + public void testDiffIndexUpdate() throws Exception { + // Create a memory node store + NodeStore store = new MemoryNodeStore(INITIAL_CONTENT); + + storeDiff(store, "2026-01-01T00:00:00.000Z", "" + + "{ \"acme.testIndex\": {\n" + + " \"async\": [ \"async\", \"nrt\" ],\n" + + " \"compatVersion\": 2,\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"includedPaths\": [ \"/content/dam\" ],\n" + + " \"jcr:primaryType\": \"oak:QueryIndexDefinition\",\n" + + " \"queryPaths\": [ \"/content/dam\" ],\n" + + " \"selectionPolicy\": \"tag\",\n" + + " \"tags\": [ \"abc\" ],\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"created\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"name\": \"str:jcr:created\",\n" + + " \"ordered\": true,\n" + + " \"propertyIndex\": true,\n" + + " \"type\": \"Date\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " } }"); + + JsonObject repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(store, "lucene"); + assertSameJson("{\n" + + " \"/oak:index/acme.testIndex-1-custom-1\": {\n" + + " \"compatVersion\": 2,\n" + + " \"async\": [\"async\", \"nrt\"],\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"mergeChecksum\": \"34e7f7f0eb480ea781317b56134bc85fc59ed97031d95f518fdcff230aec28a2\",\n" + + " \"mergeInfo\": \"This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html\",\n" + + " \"selectionPolicy\": \"tag\",\n" + + " \"queryPaths\": [\"/content/dam\"],\n" + + " \"includedPaths\": [\"/content/dam\"],\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"tags\": [\"abc\"],\n" + + " \"merges\": [\"/oak:index/acme.testIndex\"],\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"created\": {\n" + + " \"ordered\": true,\n" + + " \"name\": \"str:jcr:created\",\n" + + " \"propertyIndex\": true,\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"type\": \"Date\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", repositoryDefinitions.toString()); + + storeDiff(store, "2026-01-01T00:00:00.001Z", "" + + "{ \"acme.testIndex\": {\n" + + " \"async\": [ \"async\", \"nrt\" ],\n" + + " \"compatVersion\": 2,\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"includedPaths\": [ \"/content/dam\" ],\n" + + " \"jcr:primaryType\": \"oak:QueryIndexDefinition\",\n" + + " \"queryPaths\": [ \"/content/dam\" ],\n" + + " \"selectionPolicy\": \"tag\",\n" + + " \"tags\": [ \"abc\" ],\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"created\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"name\": \"str:jcr:created\",\n" + + " \"propertyIndex\": true\n" + + " },\n" + + " \"modified\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"name\": \"str:jcr:modified\",\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " } }"); + + repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(store, "lucene"); + assertSameJson("{\n" + + " \"/oak:index/acme.testIndex-1-custom-2\": {\n" + + " \"compatVersion\": 2,\n" + + " \"async\": [\"async\", \"nrt\"],\n" + + " \"mergeChecksum\": \"41df9c87e4d4fca446aed3f55e6d188304a2cb49bae442b75403dc23a89b266f\",\n" + + " \"mergeInfo\": \"This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html\",\n" + + " \"selectionPolicy\": \"tag\",\n" + + " \"queryPaths\": [\"/content/dam\"],\n" + + " \"includedPaths\": [\"/content/dam\"],\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"type\": \"lucene\",\n" + + " \"tags\": [\"abc\"],\n" + + " \"merges\": [\"/oak:index/acme.testIndex\"],\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"created\": {\n" + + " \"name\": \"str:jcr:created\",\n" + + " \"propertyIndex\": true,\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\"\n" + + " },\n" + + " \"modified\": {\n" + + " \"name\": \"str:jcr:modified\",\n" + + " \"propertyIndex\": true,\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", repositoryDefinitions.toString()); + + storeDiff(store, "2026-01-01T00:00:00.002Z", "" + + "{}"); + + repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(store, "lucene"); + assertSameJson("{}", repositoryDefinitions.toString()); + } + + private void assertSameJson(String a, String b) { + JsonObject ja = JsonObject.fromJson(a, true); + JsonObject jb = JsonObject.fromJson(b, true); + if (!DiffIndexMerger.isSameIgnorePropertyOrder(ja, jb)) { + assertEquals(a, b); + } + } + + private void storeDiff(NodeStore store, String timestamp, String json) throws CommitFailedException { + // Get the root builder + NodeBuilder builder = store.getRoot().builder(); + + List indexEditors = List.of( + new ReferenceEditorProvider(), new PropertyIndexEditorProvider(), new NodeCounterEditorProvider()); + IndexEditorProvider provider = CompositeIndexEditorProvider.compose(indexEditors); + EditorHook hook = new EditorHook(new IndexUpdateProvider(provider)); + + // Create the index definition at /oak:index/diff.index + NodeBuilder indexDefs = builder.child(INDEX_DEFINITIONS_NAME); + NodeBuilder diffIndex = indexDefs.child("diff.index"); + + // Set index properties + diffIndex.setProperty("jcr:primaryType", IndexConstants.INDEX_DEFINITIONS_NODE_TYPE, Type.NAME); + diffIndex.setProperty(TYPE_PROPERTY_NAME, "disabled"); + + // Create the diff.json child node with primary type nt:file + NodeBuilder diffJson = diffIndex.child("diff.json"); + diffJson.setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_FILE, Type.NAME); + + // Create jcr:content child node (required for nt:file) with empty text + NodeBuilder content = diffJson.child(JcrConstants.JCR_CONTENT); + content.setProperty(JcrConstants.JCR_LASTMODIFIED, timestamp); + content.setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_RESOURCE, Type.NAME); + + content.setProperty("jcr:data", json); + + // Merge changes to the store + store.merge(builder, hook, CommitInfo.EMPTY); + + // Run async indexing explicitly + for (int i = 0; i < 5; i++) { + try (AsyncIndexUpdate async = new AsyncIndexUpdate("async", store, provider)) { + async.run(); + } + } + } +} + diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java new file mode 100644 index 00000000000..9a80fa8a7e1 --- /dev/null +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import java.util.TreeSet; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.json.JsonUtils; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.junit.Test; + +public class JsonNodeBuilderTest { + + @Test + public void addNodeTypeAndUUID() throws CommitFailedException, IOException { + MemoryNodeStore ns = new MemoryNodeStore(); + JsonObject json = JsonObject.fromJson( + "{\n" + + " \"includedPaths\": \"/same\",\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"queryPaths\": \"/same\",\n" + + " \"type\": \"lucene\",\n" + + " \"diff.json\": {\n" + + " \"jcr:primaryType\": \"nt:file\",\n" + + " \"jcr:content\": {\n" + + " \"jcr:data\": \":blobId:dGVzdA==\",\n" + + " \"jcr:mimeType\": \"application/json\",\n" + + " \"jcr:primaryType\": \"nt:resource\"\n" + + " }\n" + + " }\n" + + " }", true); + NodeBuilder builder = ns.getRoot().builder(); + JsonNodeBuilder.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); + ns.merge(builder, new EmptyHook(), CommitInfo.EMPTY); + String json2 = JsonUtils.nodeStateToJson(ns.getRoot(), 5); + json2 = json2.replaceAll("jcr:uuid\" : \".*\"", "jcr:uuid\" : \"...\""); + assertEquals("{\n" + + " \"test\" : {\n" + + " \"queryPaths\" : \"/same\",\n" + + " \"includedPaths\" : \"/same\",\n" + + " \"jcr:primaryType\" : \"nt:unstructured\",\n" + + " \"type\" : \"lucene\",\n" + + " \":childOrder\" : [ \"diff.json\" ],\n" + + " \"diff.json\" : {\n" + + " \"jcr:primaryType\" : \"nt:file\",\n" + + " \":childOrder\" : [ \"jcr:content\" ],\n" + + " \"jcr:content\" : {\n" + + " \"jcr:mimeType\" : \"application/json\",\n" + + " \"jcr:data\" : \"test\",\n" + + " \"jcr:primaryType\" : \"nt:resource\",\n" + + " \"jcr:uuid\" : \"...\",\n" + + " \":childOrder\" : [ ]\n" + + " }\n" + + " }\n" + + " }\n" + + "}", json2); + + json = JsonObject.fromJson( + "{\"number\":1," + + "\"double2\":1.0," + + "\"child2\":{\"y\":2}}", true); + builder = ns.getRoot().builder(); + JsonNodeBuilder.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); + ns.merge(builder, new EmptyHook(), CommitInfo.EMPTY); + assertEquals("{\n" + + " \"test\" : {\n" + + " \"number\" : 1,\n" + + " \"double2\" : 1.0,\n" + + " \"jcr:primaryType\" : \"nt:test\",\n" + + " \":childOrder\" : [ \"child2\" ],\n" + + " \"child2\" : {\n" + + " \"y\" : 2,\n" + + " \"jcr:primaryType\" : \"nt:test\",\n" + + " \":childOrder\" : [ ]\n" + + " }\n" + + " }\n" + + "}", JsonUtils.nodeStateToJson(ns.getRoot(), 5)); + } + + @Test + public void store() throws CommitFailedException, IOException { + MemoryNodeStore ns = new MemoryNodeStore(); + JsonObject json = JsonObject.fromJson( + "{\"number\":1," + + "\"double\":1.0," + + "\"string\":\"hello\"," + + "\"array\":[\"a\",\"b\"]," + + "\"child\":{\"x\":1}," + + "\"blob\":\":blobId:dGVzdA==\"}", true); + NodeBuilder builder = ns.getRoot().builder(); + JsonNodeBuilder.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); + ns.merge(builder, new EmptyHook(), CommitInfo.EMPTY); + assertEquals("{\n" + + " \"test\" : {\n" + + " \"number\" : 1,\n" + + " \"blob\" : \"test\",\n" + + " \"string\" : \"hello\",\n" + + " \"array\" : [ \"a\", \"b\" ],\n" + + " \"double\" : 1.0,\n" + + " \"jcr:primaryType\" : \"nt:test\",\n" + + " \":childOrder\" : [ \"child\" ],\n" + + " \"child\" : {\n" + + " \"x\" : 1,\n" + + " \"jcr:primaryType\" : \"nt:test\",\n" + + " \":childOrder\" : [ ]\n" + + " }\n" + + " }\n" + + "}", JsonUtils.nodeStateToJson(ns.getRoot(), 5)); + + json = JsonObject.fromJson( + "{\"number\":1," + + "\"double2\":1.0," + + "\"child2\":{\"y\":2}}", true); + builder = ns.getRoot().builder(); + JsonNodeBuilder.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); + ns.merge(builder, new EmptyHook(), CommitInfo.EMPTY); + assertEquals("{\n" + + " \"test\" : {\n" + + " \"number\" : 1,\n" + + " \"double2\" : 1.0,\n" + + " \"jcr:primaryType\" : \"nt:test\",\n" + + " \":childOrder\" : [ \"child2\" ],\n" + + " \"child2\" : {\n" + + " \"y\" : 2,\n" + + " \"jcr:primaryType\" : \"nt:test\",\n" + + " \":childOrder\" : [ ]\n" + + " }\n" + + " }\n" + + "}", JsonUtils.nodeStateToJson(ns.getRoot(), 5)); + } + + @Test + public void oakStringValue() { + assertEquals("123", JsonNodeBuilder.oakStringValue("123")); + assertEquals("45.67", JsonNodeBuilder.oakStringValue("45.67")); + assertEquals("-10", JsonNodeBuilder.oakStringValue("-10")); + + String helloBase64 = Base64.getEncoder().encodeToString("hello".getBytes(StandardCharsets.UTF_8)); + assertEquals("hello", JsonNodeBuilder.oakStringValue("\":blobId:" + helloBase64 + "\"")); + + assertEquals("hello", JsonNodeBuilder.oakStringValue("\"str:hello\"")); + assertEquals("acme:Test", JsonNodeBuilder.oakStringValue("\"nam:acme:Test\"")); + assertEquals("2024-01-19", JsonNodeBuilder.oakStringValue("\"dat:2024-01-19\"")); + } + + @Test + public void getStringSet() { + assertNull(JsonNodeBuilder.getStringSet(null)); + assertEquals(new TreeSet<>(Arrays.asList("hello")), JsonNodeBuilder.getStringSet("\"hello\"")); + assertEquals(null, JsonNodeBuilder.getStringSet("123")); + assertEquals(new TreeSet<>(Arrays.asList("content/abc")), JsonNodeBuilder.getStringSet("\"content\\/abc\"")); + assertTrue(JsonNodeBuilder.getStringSet("[]").isEmpty()); + assertEquals(new TreeSet<>(Arrays.asList("a")), JsonNodeBuilder.getStringSet("[\"a\"]")); + assertEquals(new TreeSet<>(Arrays.asList("content/abc")), JsonNodeBuilder.getStringSet("[\"content\\/abc\"]")); + assertEquals(new TreeSet<>(Arrays.asList("a")), JsonNodeBuilder.getStringSet("[\"a\",\"a\"]")); + assertEquals(new TreeSet<>(Arrays.asList("a", "z")), JsonNodeBuilder.getStringSet("[\"z\",\"a\"]")); + } + + @Test + public void oakStringArrayValue() throws IOException { + assertNull(JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{}", true), "p")); + assertArrayEquals(new String[]{"hello"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":\"hello\"}", true), "p")); + assertNull(JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":123}", true), "p")); + assertArrayEquals(new String[]{"content/abc"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":\"content\\/abc\"}", true), "p")); + assertArrayEquals(new String[]{}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":[]}", true), "p")); + assertArrayEquals(new String[]{"a"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"a\"]}", true), "p")); + assertArrayEquals(new String[]{"content/abc"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"content\\/abc\"]}", true), "p")); + assertArrayEquals(new String[]{"a"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"a\",\"a\"]}", true), "p")); + assertArrayEquals(new String[]{"a", "z"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"z\",\"a\"]}", true), "p")); + } + + @Test + public void addOrReplacePrefixesBooleansAndEscapes() throws CommitFailedException, IOException { + MemoryNodeStore ns = new MemoryNodeStore(); + JsonObject json = JsonObject.fromJson( + "{\"strValue\":\"str:hello\"," + + "\"namValue\":\"nam:acme:Test\"," + + "\"datValue\":\"dat:2024-01-19\"," + + "\"boolTrue\":true," + + "\"boolFalse\":false," + + "\"escapedArray\":[\"\\/content\\/path\"]}", true); + NodeBuilder builder = ns.getRoot().builder(); + JsonNodeBuilder.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); + ns.merge(builder, new EmptyHook(), CommitInfo.EMPTY); + assertEquals("{\n" + + " \"test\" : {\n" + + " \"namValue\" : \"acme:Test\",\n" + + " \"boolTrue\" : true,\n" + + " \"boolFalse\" : false,\n" + + " \"datValue\" : \"2024-01-19\",\n" + + " \"escapedArray\" : [ \"/content/path\" ],\n" + + " \"jcr:primaryType\" : \"nt:test\",\n" + + " \"strValue\" : \"hello\",\n" + + " \":childOrder\" : [ ]\n" + + " }\n" + + "}", JsonUtils.nodeStateToJson(ns.getRoot(), 5)); + } + +} diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java new file mode 100644 index 00000000000..35c4c414979 --- /dev/null +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import static org.junit.Assert.assertEquals; + +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.junit.Test; + +public class MergeTest { + + @Test + public void renamedProperty() { + // A property might be indexed twice, by adding two children to the "properties" node + // that both have the same "name" value. + // Alternatively, they could have the same "function" value. + String merged = DiffIndexMerger.processMerge(JsonObject.fromJson("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"name\": \"test\",\n" + + " \"boost\": 1.0\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }" + + "", true), JsonObject.fromJson("{\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"def\": {\n" + + " \"name\": \"test\",\n" + + " \"boost\": 1.2\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true)).toString(); + assertEquals("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"name\": \"test\",\n" + + " \"boost\": 1.2\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", merged); + } + + @Test + public void renamedFunction() { + // A function might be indexed twice, by adding two children to the "properties" node + // that both have the same "function" value. + String merged = DiffIndexMerger.processMerge(JsonObject.fromJson("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"function\": \"upper(test)\",\n" + + " \"boost\": 1.0\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }" + + "", true), JsonObject.fromJson("{\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"def\": {\n" + + " \"function\": \"upper(test)\",\n" + + " \"boost\": 1.2\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true)).toString(); + assertEquals("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"function\": \"upper(test)\",\n" + + " \"boost\": 1.2\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", merged); + } + + @Test + public void boost() { + // - "analyzed" must not be overwritten + // - "ordered" is added + // - "boost" is overwritten + String merged = DiffIndexMerger.processMerge(JsonObject.fromJson("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"analyzed\": true,\n" + + " \"boost\": 1.0\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }" + + "", true), JsonObject.fromJson("{\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"abc\": {\n" + + " \"analyzed\": false,\n" + + " \"ordered\": true,\n" + + " \"boost\": 1.2\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true)).toString(); + assertEquals("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"analyzed\": true,\n" + + " \"boost\": 1.2,\n" + + " \"ordered\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", merged); + } +} diff --git a/oak-core/src/test/resources/org/apache/jackrabbit/oak/plugins/index/diff/indexes.json b/oak-core/src/test/resources/org/apache/jackrabbit/oak/plugins/index/diff/indexes.json new file mode 100644 index 00000000000..a5a16d0fb72 --- /dev/null +++ b/oak-core/src/test/resources/org/apache/jackrabbit/oak/plugins/index/diff/indexes.json @@ -0,0 +1,187 @@ +{ + "/oak:index/ntFolder": { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "includedPaths": [ + "/content/test" + ], + "tags": [ + "testTag1", + "testTag2" + ], + "type": "lucene", + "async": [ + "async", + "nrt" + ], + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "nt:folder": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured", + "jcrTitle": { + "jcr:primaryType": "nam:nt:unstructured", + "nodeScopeIndex": true, + "useInSuggest": true, + "useInSpellcheck": true, + "name": "str:jcr:content/jcr:title" + } + } + } + } + }, + "/oak:index/share": { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "selectionPolicy": "tag", + "includedPaths": [ + "/var/share" + ], + "tags": [ + "share" + ], + "type": "lucene", + "async": [ + "async", + "nrt" + ], + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "nt:unstructured": { + "jcr:primaryType": "nam:nt:unstructured" + } + } + }, + "/oak:index/versionStoreIndex": { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "includedPaths": [ + "/jcr:system/jcr:versionStorage" + ], + "type": "lucene", + "async": [ + "async", + "sync" + ], + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "nt:version": { + "jcr:primaryType": "nam:nt:unstructured" + }, + "nt:frozenNode": { + "jcr:primaryType": "nam:nt:unstructured" + }, + "nt:base": { + "jcr:primaryType": "nam:nt:unstructured" + } + } + }, + "/oak:index/authorizables": { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + ":version": 2, + "type": "lucene", + "async": [ + "async", + "nrt" + ], + "excludedPaths": [ + "/var", + "/jcr:system" + ], + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "rep:Authorizable": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured" + } + } + } + }, + "/oak:index/internalVerificationLucene": { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + ":version": 2, + "includedPaths": [ + "/tmp" + ], + "type": "lucene", + "async": [ + "async" + ], + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "nt:base": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured", + "verification": { + "jcr:primaryType": "nam:nt:unstructured", + "propertyIndex": true, + "name": "verification", + "type": "String" + } + } + } + } + }, + "/oak:index/ntBaseLucene-2": { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "type": "lucene", + "async": [ + "async", + "nrt" + ], + "evaluatePathRestrictions": true, + "excludedPaths": [ + "/oak:index" + ], + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "nt:base": { + "jcr:primaryType": "nam:nt:unstructured" + } + } + }, + "/oak:index/fragments": { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "selectionPolicy": "tag", + "includedPaths": [ + "/content/dam", + "/content/launches" + ], + "tags": [ + "fragments" + ], + "type": "lucene", + "async": [ + "async", + "nrt" + ], + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured", + "properties": { + "jcr:primaryType": "nam:nt:unstructured" + } + } + } + }, + "/oak:index/assetLucene": { + "jcr:primaryType": "nam:oak:QueryIndexDefinition", + "includedPaths": [ + "/content/dam", + "/content/assets" + ], + "tags": [], + "type": "lucene", + "async": [ + "async", + "nrt" + ], + "indexRules": { + "jcr:primaryType": "nam:nt:unstructured", + "dam:Asset": { + "jcr:primaryType": "nam:nt:unstructured" + } + } + } +} diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java index f8beab7a3ba..0d47a4d148b 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java @@ -118,6 +118,8 @@ public List getPlans(Filter filter, List sortOrder, NodeS .collectIndexNodePaths(filter); if (filterReplacedIndexes()) { indexPaths = IndexName.filterReplacedIndexes(indexPaths, rootState, runIsActiveIndexCheck()); + } else { + indexPaths = IndexName.filterNewestIndexes(indexPaths, rootState); } List plans = new ArrayList<>(indexPaths.size()); for (String path : indexPaths) { diff --git a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexNameTest.java b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexNameTest.java index 1135772defb..cdcd14fe1bc 100644 --- a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexNameTest.java +++ b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexNameTest.java @@ -30,6 +30,9 @@ import org.junit.Test; import org.slf4j.event.Level; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; /** * Test the IndexName class @@ -113,4 +116,54 @@ public void recursiveActive() { lc.finished(); } } + + @Test + public void filterNewestIndexes() { + NodeState root = EMPTY_NODE; + + // Single index - should return as-is + Collection single = Arrays.asList("/lucene"); + Collection result = IndexName.filterNewestIndexes(single, root); + assertEquals(1, result.size()); + assertTrue(result.contains("/lucene")); + + // Multiple versions of the same base index - should return only the newest + Collection multipleVersions = Arrays.asList( + "/lucene", + "/lucene-1", + "/lucene-2", + "/lucene-1-custom-1", + "/lucene-2-custom-3" + ); + result = IndexName.filterNewestIndexes(multipleVersions, root); + assertEquals(1, result.size()); + assertTrue(result.contains("/lucene-2-custom-3")); + + // Different base indexes - should return newest of each + Collection differentBases = Arrays.asList( + "/luceneA", + "/luceneA-1", + "/luceneB", + "/luceneB-2-custom-1", + "/luceneC-1-custom-5" + ); + result = IndexName.filterNewestIndexes(differentBases, root); + assertEquals(new HashSet<>(Arrays.asList("/luceneA-1", "/luceneB-2-custom-1", "/luceneC-1-custom-5")), + new HashSet<>(result)); + + // Custom versions without product version + Collection customOnly = Arrays.asList( + "/lucene-custom-1", + "/lucene-custom-2", + "/lucene-custom-3" + ); + result = IndexName.filterNewestIndexes(customOnly, root); + assertEquals(1, result.size()); + assertTrue(result.contains("/lucene-custom-3")); + + // Empty collection + Collection empty = Arrays.asList(); + result = IndexName.filterNewestIndexes(empty, root); + assertTrue(result.isEmpty()); + } } From 42d1e36e4b4ef9ab5b954b6358ae3dd946fd6312 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 20 Jan 2026 11:09:45 +0100 Subject: [PATCH 02/16] OAK-12010 Simplified index management --- .../plugins/index/diff/DiffIndexMerger.java | 6 +- .../oak/plugins/index/diff/MergeTest.java | 161 ++++++++++++++++++ 2 files changed, 164 insertions(+), 3 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java index b5c45279769..b78eefbad84 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java @@ -161,7 +161,7 @@ static boolean mergeDiff(JsonObject newImageLuceneDefinitions, JsonObject combin * @param target the target map of diff.index definitions * @return the error message trying to parse the JSON file, or null */ - static String tryExtractDiffIndex(JsonObject indexDefs, String name, HashMap target) { + public static String tryExtractDiffIndex(JsonObject indexDefs, String name, HashMap target) { JsonObject diffIndex = indexDefs.getChildren().get(name); if (diffIndex == null) { return null; @@ -234,7 +234,7 @@ private static void extractExistingMergedIndexes(JsonObject indexDefs, HashMap(indexDef.getProperties().keySet())) { if (!p.endsWith("@lucene")) { diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java index 35c4c414979..28a689fdc14 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java @@ -18,11 +18,84 @@ import static org.junit.Assert.assertEquals; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; + import org.apache.jackrabbit.oak.commons.json.JsonObject; import org.junit.Test; public class MergeTest { + // test that we can extract the file from the diff.json node (just that) + @Test + public void extractFile() { + JsonObject indexDiff = JsonObject.fromJson("{\n" + + " \"damAssetLucene\": {\n" + + " \"indexRules\": {\n" + + " \"dam:Asset\": {\n" + + " \"properties\": {\n" + + " \"y\": {\n" + + " \"name\": \"y\",\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true); + String indexDiffString = indexDiff.toString(); + String base64Prop = + "\":blobId:" + Base64.getEncoder().encodeToString(indexDiffString.getBytes(StandardCharsets.UTF_8)) + "\""; + JsonObject repositoryDefinitions = JsonObject.fromJson("{\n" + + " \"/oak:index/damAssetLucene-12\": {\n" + + " \"jcr:primaryType\": \"oak:IndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"async\": [\"async\", \"nrt\"],\n" + + " \"tags\": [\"abc\"],\n" + + " \"includedPaths\": \"/content/dam\",\n" + + " \"indexRules\": {\n" + + " \"dam:Asset\": {\n" + + " \"properties\": {\n" + + " \"x\": {\n" + + " \"name\": \"x\",\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"/oak:index/diff.index\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"type\": \"lucene\", \"includedPaths\": \"/same\", \"queryPaths\": \"/same\",\n" + + " \"diff.json\": {\n" + + " \"jcr:primaryType\": \"nam:nt:file\",\n" + + " \"jcr:content\": {\n" + + " \"jcr:primaryType\": \"nam:nt:resource\",\n" + + " \"jcr:mimeType\": \"application/json\",\n" + + " \"jcr:data\":\n" + + " " + base64Prop + "\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true); + + HashMap target = new HashMap<>(); + DiffIndexMerger.tryExtractDiffIndex(repositoryDefinitions, "/oak:index/diff.index", target); + assertEquals("{damAssetLucene={\n" + + " \"indexRules\": {\n" + + " \"dam:Asset\": {\n" + + " \"properties\": {\n" + + " \"y\": {\n" + + " \"name\": \"y\",\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}}", target.toString()); + } + @Test public void renamedProperty() { // A property might be indexed twice, by adding two children to the "properties" node @@ -188,4 +261,92 @@ public void boost() { + " }\n" + "}", merged); } + + @Test + public void mergeDiffsTest() { + JsonObject a = JsonObject.fromJson("{\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"prop1\": {\n" + + " \"name\": \"field1\",\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": \"lucene\"\n" + + " }", true); + JsonObject b = JsonObject.fromJson("{\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"prop2\": {\n" + + " \"name\": \"field2\",\n" + + " \"ordered\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"async\": [\"async\", \"nrt\"]\n" + + " }", true); + String merged = DiffIndexMerger.mergeDiffs(a, b).toString(); + assertEquals("{\n" + + " \"type\": \"lucene\",\n" + + " \"async\": [\"async\", \"nrt\"],\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"prop1\": {\n" + + " \"name\": \"field1\",\n" + + " \"propertyIndex\": true\n" + + " },\n" + + " \"prop2\": {\n" + + " \"name\": \"field2\",\n" + + " \"ordered\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", merged); + } + + @Test + public void switchToLuceneChildrenTest() { + JsonObject indexDef = JsonObject.fromJson("{\n" + + " \"type\": \"elasticsearch\",\n" + + " \"type@lucene\": \"lucene\",\n" + + " \"async@lucene\": \"[\\\"async\\\", \\\"nrt\\\"]\",\n" + + " \"async\": \"[\\\"async\\\"]\",\n" + + " \"codec@lucene\": \"Lucene46\",\n" + + " \"indexRules\": {\n" + + " \"dam:Asset\": {\n" + + " \"properties\": {\n" + + " \"test\": {\n" + + " \"name\": \"jcr:content/metadata/test\",\n" + + " \"boost@lucene\": \"2.0\",\n" + + " \"boost\": \"1.0\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true); + DiffIndexMerger.switchToLuceneChildren(indexDef); + String result = indexDef.toString(); + assertEquals("{\n" + + " \"type\": \"lucene\",\n" + + " \"async\": \"[\\\"async\\\", \\\"nrt\\\"]\",\n" + + " \"codec\": \"Lucene46\",\n" + + " \"indexRules\": {\n" + + " \"dam:Asset\": {\n" + + " \"properties\": {\n" + + " \"test\": {\n" + + " \"name\": \"jcr:content/metadata/test\",\n" + + " \"boost\": \"2.0\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", result); + } } From 041af37ef04a75208393898082d60c334941f989 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 20 Jan 2026 15:15:28 +0100 Subject: [PATCH 03/16] OAK-12010 Simplified index management --- .../oak/plugins/index/diff/DiffIndex.java | 2 +- .../plugins/index/diff/DiffIndexMerger.java | 42 ++++++--- .../oak/plugins/index/diff/MergeTest.java | 91 ++++++++++++++++++- 3 files changed, 118 insertions(+), 17 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java index 825a23c38ea..19a8b563994 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -109,7 +109,7 @@ public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefin JsonObject repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(indexDefinitions); LOG.debug("Index list {}", repositoryDefinitions.toString()); try { - DiffIndexMerger.merge(newImageLuceneDefinitions, repositoryDefinitions, store); + DiffIndexMerger.instance().merge(newImageLuceneDefinitions, repositoryDefinitions, store); for (String indexPath : newImageLuceneDefinitions.getChildren().keySet()) { if (indexPath.startsWith("/oak:index/" + DiffIndexMerger.DIFF_INDEX)) { continue; diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java index b78eefbad84..47c530ff06f 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java @@ -62,6 +62,22 @@ public class DiffIndexMerger { // in case a customization was removed, create a copy of the OOTB index private final static boolean DELETE_COPIES_OOTB = Boolean.getBoolean("oak.diffIndex.deleteCopiesOOTB"); + private final String[] unsupportedIncludedPaths; + private final boolean deleteCreatesDummyIndex; + private final boolean deleteCopiesOutOfTheBoxIndex; + + static final DiffIndexMerger INSTANCE = new DiffIndexMerger(UNSUPPORTED_INCLUDED_PATHS, DELETE_CREATES_DUMMY, DELETE_COPIES_OOTB); + + public static DiffIndexMerger instance() { + return INSTANCE; + } + + DiffIndexMerger(String[] unsupportedIncludedPaths, boolean deleteCreatesDummyIndex, boolean deleteCopiesOutOfTheBoxIndex) { + this.unsupportedIncludedPaths = unsupportedIncludedPaths; + this.deleteCreatesDummyIndex = deleteCreatesDummyIndex; + this.deleteCopiesOutOfTheBoxIndex = deleteCopiesOutOfTheBoxIndex; + } + /** * If there is a diff index, that is an index with prefix "diff.", then try to merge it. * @@ -73,7 +89,7 @@ public class DiffIndexMerger { * (input) * @param repositoryNodeStore */ - public static void merge(JsonObject newImageLuceneDefinitions, JsonObject repositoryDefinitions, NodeStore repositoryNodeStore) { + public void merge(JsonObject newImageLuceneDefinitions, JsonObject repositoryDefinitions, NodeStore repositoryNodeStore) { // combine all definitions into one object JsonObject combined = new JsonObject(); @@ -117,7 +133,7 @@ public static void merge(JsonObject newImageLuceneDefinitions, JsonObject reposi * (input) * @return whether a new version of an index was added */ - static boolean mergeDiff(JsonObject newImageLuceneDefinitions, JsonObject combined) { + boolean mergeDiff(JsonObject newImageLuceneDefinitions, JsonObject combined) { // iterate again, this time process // collect the diff index(es) @@ -268,7 +284,7 @@ public static JsonObject mergeDiffs(JsonObject a, JsonObject b) { * (input) * @return whether a new version of an index was added */ - public static boolean processMerge(String indexName, JsonObject indexDiff, JsonObject newImageLuceneDefinitions, JsonObject combined) { + public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject newImageLuceneDefinitions, JsonObject combined) { // extract the latest product index (eg. damAssetLucene-12) // and customized index (eg. damAssetLucene-12-custom-3) - if any IndexName latestProduct = null; @@ -321,7 +337,7 @@ public static boolean processMerge(String indexName, JsonObject indexDiff, JsonO includedPaths = JsonNodeBuilder.oakStringArrayValue(indexDiff, "includedPaths"); if (includesUnsupportedPaths(includedPaths)) { LOG.warn("New custom index {} is not supported because it contains an unsupported path ({})", - indexName, Arrays.toString(UNSUPPORTED_INCLUDED_PATHS)); + indexName, Arrays.toString(unsupportedIncludedPaths)); return false; } } @@ -329,7 +345,7 @@ public static boolean processMerge(String indexName, JsonObject indexDiff, JsonO includedPaths = JsonNodeBuilder.oakStringArrayValue(latestProductIndex, "includedPaths"); if (includesUnsupportedPaths(includedPaths)) { LOG.warn("Customizing index {} is not supported because it contains an unsupported path ({})", - latestProductKey, Arrays.toString(UNSUPPORTED_INCLUDED_PATHS)); + latestProductKey, Arrays.toString(unsupportedIncludedPaths)); return false; } } @@ -409,7 +425,7 @@ public static boolean processMerge(String indexName, JsonObject indexDiff, JsonO merged.getProperties().put("merges", "[" + JsopBuilder.encode("/oak:index/" + indexName) + "]"); merged.getProperties().remove("reindexCount"); merged.getProperties().remove("reindex"); - if (!DELETE_COPIES_OOTB && indexDiff.toString().equals("{}")) { + if (!deleteCopiesOutOfTheBoxIndex && indexDiff.toString().equals("{}")) { merged.getProperties().put("type", "\"disabled\""); merged.getProperties().put("mergeComment", "\"This index is superseeded and can be removed\""); } @@ -425,8 +441,8 @@ public static boolean processMerge(String indexName, JsonObject indexDiff, JsonO * @param includedPaths the includedPaths list * @return true if any unsupported path is included */ - public static boolean includesUnsupportedPaths(String[] includedPaths) { - if (UNSUPPORTED_INCLUDED_PATHS.length == 1 && "".equals(UNSUPPORTED_INCLUDED_PATHS[0])) { + public boolean includesUnsupportedPaths(String[] includedPaths) { + if (unsupportedIncludedPaths.length == 1 && "".equals(unsupportedIncludedPaths[0])) { // set to an empty string return false; } @@ -439,7 +455,7 @@ public static boolean includesUnsupportedPaths(String[] includedPaths) { // all return true; } - for (String unsupported : UNSUPPORTED_INCLUDED_PATHS) { + for (String unsupported : unsupportedIncludedPaths) { if (unsupported.isEmpty()) { continue; } @@ -601,7 +617,7 @@ private static void removeUUIDs(JsonObject obj) { * @param diff the diff (from the diff.index definition) * @return the index definition of the merged index */ - public static JsonObject processMerge(JsonObject productIndex, JsonObject diff) { + public JsonObject processMerge(JsonObject productIndex, JsonObject diff) { JsonObject result; if (productIndex == null) { // fully custom index @@ -649,7 +665,7 @@ private static void addPrimaryType(String path, JsonObject json) { * @param diff the diff (what to merge) * @param target where to merge into */ - private static void mergeInto(String path, JsonObject diff, JsonObject target) { + private void mergeInto(String path, JsonObject diff, JsonObject target) { for (String p : diff.getProperties().keySet()) { if (path.isEmpty()) { if ("jcr:primaryType".equals(p)) { @@ -700,7 +716,7 @@ private static void mergeInto(String path, JsonObject diff, JsonObject target) { mergeInto(path + "/" + targetChildName, diff.getChildren().get(c), target.getChildren().get(targetChildName)); } if (target.getProperties().isEmpty() && target.getChildren().isEmpty()) { - if (DELETE_CREATES_DUMMY) { + if (deleteCreatesDummyIndex) { // dummy index target.getProperties().put("async", "\"async\""); target.getProperties().put("includedPaths", "\"/dummy\""); @@ -778,7 +794,7 @@ public static boolean isSameIgnorePropertyOrder(JsonObject a, JsonObject b) { * @param repositoryNodeStore the node store * @return a map, possibly with a single entry with this key */ - static Map readDiffIndex(NodeStore repositoryNodeStore, String name) { + public static Map readDiffIndex(NodeStore repositoryNodeStore, String name) { HashMap map = new HashMap<>(); NodeState root = repositoryNodeStore.getRoot(); String indexPath = "/oak:index/" + name; diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java index 28a689fdc14..57416435832 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java @@ -16,13 +16,22 @@ */ package org.apache.jackrabbit.oak.plugins.index.diff; +import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.HashMap; +import java.util.Map; +import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.junit.Test; public class MergeTest { @@ -101,7 +110,7 @@ public void renamedProperty() { // A property might be indexed twice, by adding two children to the "properties" node // that both have the same "name" value. // Alternatively, they could have the same "function" value. - String merged = DiffIndexMerger.processMerge(JsonObject.fromJson("{\n" + String merged = DiffIndexMerger.instance().processMerge(JsonObject.fromJson("{\n" + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + " \"type\": \"lucene\",\n" + " \"indexRules\": {\n" @@ -155,7 +164,7 @@ public void renamedProperty() { public void renamedFunction() { // A function might be indexed twice, by adding two children to the "properties" node // that both have the same "function" value. - String merged = DiffIndexMerger.processMerge(JsonObject.fromJson("{\n" + String merged = DiffIndexMerger.instance().processMerge(JsonObject.fromJson("{\n" + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + " \"type\": \"lucene\",\n" + " \"indexRules\": {\n" @@ -205,12 +214,38 @@ public void renamedFunction() { + "}", merged); } + @Test + public void createDummy() { + // when enabling "deleteCreatesDummyIndex", then a dummy index is created + // (that indexes /dummy, which doesn't exist) + String merged = new DiffIndexMerger(new String[0], true, true).processMerge(JsonObject.fromJson("{}" + + "", true), JsonObject.fromJson("{}", true)).toString(); + assertEquals("{\n" + + " \"async\": \"async\",\n" + + " \"includedPaths\": \"/dummy\",\n" + + " \"queryPaths\": \"/dummy\",\n" + + " \"type\": \"lucene\",\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"dummy\": {\n" + + " \"name\": \"dummy\",\n" + + " \"propertyIndex\": true,\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}", merged); + } + @Test public void boost() { // - "analyzed" must not be overwritten // - "ordered" is added // - "boost" is overwritten - String merged = DiffIndexMerger.processMerge(JsonObject.fromJson("{\n" + String merged = DiffIndexMerger.instance().processMerge(JsonObject.fromJson("{\n" + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + " \"type\": \"lucene\",\n" + " \"indexRules\": {\n" @@ -349,4 +384,54 @@ public void switchToLuceneChildrenTest() { + " }\n" + "}", result); } + + @Test + public void includesUnsupportedPathsTest() { + DiffIndexMerger merger = new DiffIndexMerger(new String[]{"/apps", "/libs"}, false, false); + + assertEquals(true, merger.includesUnsupportedPaths(null)); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/apps"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/apps/acme"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/apps/acme/test"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/libs"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/libs/foundation"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/content", "/apps"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/content", "/libs/test"})); + + assertEquals(false, merger.includesUnsupportedPaths(new String[]{"/content"})); + assertEquals(false, merger.includesUnsupportedPaths(new String[]{"/content/dam"})); + assertEquals(false, merger.includesUnsupportedPaths(new String[]{"/var"})); + assertEquals(false, merger.includesUnsupportedPaths(new String[]{"/etc"})); + assertEquals(false, merger.includesUnsupportedPaths(new String[]{"/content", "/var", "/etc"})); + } + + @Test + public void readDiffIndexTest() throws CommitFailedException { + NodeStore store = new MemoryNodeStore(INITIAL_CONTENT); + NodeBuilder root = store.getRoot().builder(); + NodeBuilder oakIndex = root.child("oak:index"); + NodeBuilder diffIndex = oakIndex.child("diff.index.optimizer"); + diffIndex.setProperty("jcr:primaryType", "nt:unstructured"); + diffIndex.setProperty("type", "lucene"); + diffIndex.setProperty("async", "async"); + diffIndex.setProperty("includedPaths", "/content"); + NodeBuilder indexRules = diffIndex.child("indexRules"); + NodeBuilder damAsset = indexRules.child("dam:Asset"); + NodeBuilder properties = damAsset.child("properties"); + NodeBuilder testProp = properties.child("test"); + testProp.setProperty("name", "jcr:content/metadata/test"); + testProp.setProperty("propertyIndex", true); + store.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY); + + Map result = DiffIndexMerger.readDiffIndex(store, "diff.index.optimizer"); + + assertEquals(1, result.size()); + assertTrue(result.containsKey("/oak:index/diff.index.optimizer")); + JsonObject indexDef = result.get("/oak:index/diff.index.optimizer"); + assertEquals("\"lucene\"", indexDef.getProperties().get("type")); + assertEquals("\"async\"", indexDef.getProperties().get("async")); + assertEquals("\"/content\"", indexDef.getProperties().get("includedPaths")); + assertTrue(indexDef.getChildren().containsKey("indexRules")); + } } From e2d6c7274c9c8209d807fee8ff373f6fc96569b8 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Wed, 21 Jan 2026 08:50:47 +0100 Subject: [PATCH 04/16] OAK-12010 Simplified index management --- .../oak/plugins/index/diff/DiffIndex.java | 16 ++---- .../plugins/index/diff/DiffIndexMerger.java | 56 ++++++++++++------- .../oak/plugins/index/diff/DiffIndexTest.java | 2 +- .../oak/plugins/index/diff/MergeTest.java | 6 +- 4 files changed, 46 insertions(+), 34 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java index 19a8b563994..6ee23643ea0 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -61,20 +61,16 @@ public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefin continue; } NodeBuilder diffIndexDefinition = indexDefinitions.child(diffIndex); - NodeBuilder diffJson = diffIndexDefinition.getChildNode("diff.json"); - if (!diffJson.exists()) { + NodeBuilder diffContent = diffIndexDefinition.getChildNode("diff.json").getChildNode("jcr:content"); + if (!diffContent.exists()) { continue; } - NodeBuilder jcrContent = diffJson.getChildNode("jcr:content"); - if (!jcrContent.exists()) { - continue; - } - PropertyState lastMod = jcrContent.getProperty("jcr:lastModified"); + PropertyState lastMod = diffContent.getProperty("jcr:lastModified"); if (lastMod == null) { continue; } String modified = lastMod.getValue(Type.DATE); - PropertyState lastProcessed = jcrContent.getProperty(":lastProcessed"); + PropertyState lastProcessed = diffContent.getProperty(":lastProcessed"); if (lastProcessed != null) { if (modified.equals(lastProcessed.getValue(Type.STRING))) { // already processed @@ -82,8 +78,8 @@ public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefin } } // store now, so a change is only processed once - jcrContent.setProperty(":lastProcessed", modified); - PropertyState jcrData = jcrContent.getProperty("jcr:data"); + diffContent.setProperty(":lastProcessed", modified); + PropertyState jcrData = diffContent.getProperty("jcr:data"); String diff = tryReadString(jcrData); if (diff == null) { continue; diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java index 47c530ff06f..f6d1ebc9c4f 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java @@ -45,7 +45,7 @@ */ public class DiffIndexMerger { - final static Logger LOG = LoggerFactory.getLogger(DiffIndexMerger.class); + private final static Logger LOG = LoggerFactory.getLogger(DiffIndexMerger.class); public final static String DIFF_INDEX = "diff.index"; public final static String DIFF_INDEX_OPTIMIZER = "diff.index.optimizer"; @@ -62,20 +62,28 @@ public class DiffIndexMerger { // in case a customization was removed, create a copy of the OOTB index private final static boolean DELETE_COPIES_OOTB = Boolean.getBoolean("oak.diffIndex.deleteCopiesOOTB"); + // whether to log at info level + private final static boolean LOG_AT_INFO_LEVEL = Boolean.getBoolean("oak.diffIndex.logAtInfoLevel"); + private final String[] unsupportedIncludedPaths; private final boolean deleteCreatesDummyIndex; private final boolean deleteCopiesOutOfTheBoxIndex; + private final boolean logAtInfoLevel; - static final DiffIndexMerger INSTANCE = new DiffIndexMerger(UNSUPPORTED_INCLUDED_PATHS, DELETE_CREATES_DUMMY, DELETE_COPIES_OOTB); + static final DiffIndexMerger INSTANCE = new DiffIndexMerger(UNSUPPORTED_INCLUDED_PATHS, + DELETE_CREATES_DUMMY, DELETE_COPIES_OOTB, LOG_AT_INFO_LEVEL); public static DiffIndexMerger instance() { return INSTANCE; } - DiffIndexMerger(String[] unsupportedIncludedPaths, boolean deleteCreatesDummyIndex, boolean deleteCopiesOutOfTheBoxIndex) { + DiffIndexMerger(String[] unsupportedIncludedPaths, + boolean deleteCreatesDummyIndex, boolean deleteCopiesOutOfTheBoxIndex, + boolean logAtInfoLevel) { this.unsupportedIncludedPaths = unsupportedIncludedPaths; this.deleteCreatesDummyIndex = deleteCreatesDummyIndex; this.deleteCopiesOutOfTheBoxIndex = deleteCopiesOutOfTheBoxIndex; + this.logAtInfoLevel = logAtInfoLevel; } /** @@ -113,7 +121,7 @@ public void merge(JsonObject newImageLuceneDefinitions, JsonObject repositoryDef if (!found) { // early exit, so that the risk of merging the PR // is very small for customers that do not use this - LOG.debug("No 'diff.index' definition"); + log("No 'diff.index' definition"); return; } mergeDiff(newImageLuceneDefinitions, combined); @@ -144,7 +152,7 @@ boolean mergeDiff(JsonObject newImageLuceneDefinitions, JsonObject combined) { // (indexes with mergeInfo), then we need to disable those (using /dummy includedPath) extractExistingMergedIndexes(combined, toProcess); if (toProcess.isEmpty()) { - LOG.debug("No diff index definitions found."); + log("No diff index definitions found."); return false; } boolean hasChanges = false; @@ -155,7 +163,7 @@ boolean mergeDiff(JsonObject newImageLuceneDefinitions, JsonObject combined) { LOG.warn("The key should contains just the index name, without the '/oak:index' prefix for key {}", key); key = key.substring("/oak:index/".length()); } - LOG.debug("Processing {}", key); + log("Processing {}", key); hasChanges |= processMerge(key, value, newImageLuceneDefinitions, combined); } return hasChanges; @@ -295,7 +303,7 @@ public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject n for (String key : combined.getChildren().keySet()) { IndexName name = IndexName.parse(key.substring(prefix.length())); if (!name.isVersioned()) { - LOG.debug("Ignoring unversioned index {}", name); + log("Ignoring unversioned index {}", name); continue; } if (!name.getBaseName().equals(indexName)) { @@ -316,14 +324,14 @@ public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject n } } } - LOG.debug("Latest product: {}", latestProductKey); - LOG.debug("Latest customized: {}", latestCustomizedKey); + log("Latest product: {}", latestProductKey); + log("Latest customized: {}", latestCustomizedKey); if (latestProduct == null) { if (indexName.indexOf('.') >= 0) { // a fully custom index needs to contains a dot - LOG.debug("Fully custom index {}", indexName); + log("Fully custom index {}", indexName); } else { - LOG.debug("No product version for {}", indexName); + log("No product version for {}", indexName); return false; } } @@ -332,7 +340,7 @@ public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject n if (latestProductIndex == null) { if (indexDiff.getProperties().isEmpty() && indexDiff.getChildren().isEmpty()) { // there is no customization (any more), which means a dummy index may be needed - LOG.debug("No customization for {}", indexName); + log("No customization for {}", indexName); } else { includedPaths = JsonNodeBuilder.oakStringArrayValue(indexDiff, "includedPaths"); if (includesUnsupportedPaths(includedPaths)) { @@ -355,7 +363,7 @@ public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject n if (indexDiff == null) { // no diff definition: use to the OOTB index if (latestCustomized == null) { - LOG.debug("Only a product index found, nothing to do"); + log("Only a product index found, nothing to do"); return false; } merged = latestProductIndex; @@ -384,7 +392,7 @@ public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject n if (isSameIgnorePropertyOrder(mergedDef, latestDef)) { // normal case: no change // (even if checksums do not match: checksums might be missing or manipulated) - LOG.debug("Latest index matches"); + log("Latest index matches"); if (latestMergeChecksum != null && !latestMergeChecksum.equals(mergeChecksum)) { LOG.warn("Indexes do match, but checksums do not. Possibly checksum was changed: {} vs {}", latestMergeChecksum, mergeChecksum); LOG.warn("latest: {}\nmerged: {}", latestDef, mergedDef); @@ -766,9 +774,9 @@ public static String getChildWithKeyValuePair(JsonObject obj, String key, String * @param b the second object * @return true if the keys and values are equal */ - public static boolean isSameIgnorePropertyOrder(JsonObject a, JsonObject b) { + public boolean isSameIgnorePropertyOrder(JsonObject a, JsonObject b) { if (!a.getChildren().keySet().equals(b.getChildren().keySet())) { - LOG.debug("Child (order) difference: {} vs {}", + log("Child (order) difference: {} vs {}", a.getChildren().keySet(), b.getChildren().keySet()); return false; } @@ -781,7 +789,7 @@ public static boolean isSameIgnorePropertyOrder(JsonObject a, JsonObject b) { TreeMap pa = new TreeMap<>(a.getProperties()); TreeMap pb = new TreeMap<>(b.getProperties()); if (!pa.toString().equals(pb.toString())) { - LOG.debug("Property value difference: {} vs {}", pa.toString(), pb.toString()); + log("Property value difference: {} vs {}", pa.toString(), pb.toString()); } return pa.toString().equals(pb.toString()); } @@ -794,12 +802,12 @@ public static boolean isSameIgnorePropertyOrder(JsonObject a, JsonObject b) { * @param repositoryNodeStore the node store * @return a map, possibly with a single entry with this key */ - public static Map readDiffIndex(NodeStore repositoryNodeStore, String name) { + public Map readDiffIndex(NodeStore repositoryNodeStore, String name) { HashMap map = new HashMap<>(); NodeState root = repositoryNodeStore.getRoot(); String indexPath = "/oak:index/" + name; NodeState idxState = NodeStateUtils.getNode(root, indexPath); - LOG.debug("Searching index {}: found={}", indexPath, idxState.exists()); + log("Searching index {}: found={}", indexPath, idxState.exists()); if (!idxState.exists()) { return map; } @@ -809,9 +817,17 @@ public static Map readDiffIndex(NodeStore repositoryNodeStor serializer.serialize(idxState); JsonObject jsonObj = JsonObject.fromJson(builder.toString(), true); jsonObj = cleanedAndNormalized(jsonObj); - LOG.debug("Found {}", jsonObj.toString()); + log("Found {}", jsonObj.toString()); map.put(indexPath, jsonObj); return map; } + private void log(String format, Object... arguments) { + if (logAtInfoLevel) { + LOG.info(format, arguments); + } else { + LOG.debug(format, arguments); + } + } + } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java index 888eeaf8983..e6a5823599e 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java @@ -260,7 +260,7 @@ public void testDiffIndexUpdate() throws Exception { private void assertSameJson(String a, String b) { JsonObject ja = JsonObject.fromJson(a, true); JsonObject jb = JsonObject.fromJson(b, true); - if (!DiffIndexMerger.isSameIgnorePropertyOrder(ja, jb)) { + if (!DiffIndexMerger.instance().isSameIgnorePropertyOrder(ja, jb)) { assertEquals(a, b); } } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java index 57416435832..c55d1c3ab6c 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java @@ -218,7 +218,7 @@ public void renamedFunction() { public void createDummy() { // when enabling "deleteCreatesDummyIndex", then a dummy index is created // (that indexes /dummy, which doesn't exist) - String merged = new DiffIndexMerger(new String[0], true, true).processMerge(JsonObject.fromJson("{}" + String merged = new DiffIndexMerger(new String[0], true, true, false).processMerge(JsonObject.fromJson("{}" + "", true), JsonObject.fromJson("{}", true)).toString(); assertEquals("{\n" + " \"async\": \"async\",\n" @@ -387,7 +387,7 @@ public void switchToLuceneChildrenTest() { @Test public void includesUnsupportedPathsTest() { - DiffIndexMerger merger = new DiffIndexMerger(new String[]{"/apps", "/libs"}, false, false); + DiffIndexMerger merger = new DiffIndexMerger(new String[]{"/apps", "/libs"}, false, false, false); assertEquals(true, merger.includesUnsupportedPaths(null)); assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/"})); @@ -424,7 +424,7 @@ public void readDiffIndexTest() throws CommitFailedException { testProp.setProperty("propertyIndex", true); store.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY); - Map result = DiffIndexMerger.readDiffIndex(store, "diff.index.optimizer"); + Map result = DiffIndexMerger.instance().readDiffIndex(store, "diff.index.optimizer"); assertEquals(1, result.size()); assertTrue(result.containsKey("/oak:index/diff.index.optimizer")); From a1c2fa9106e7b6fb6e68ce82dc5f2dfb74ebea98 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 2 Feb 2026 16:18:46 +0100 Subject: [PATCH 05/16] Update oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java Co-authored-by: Benjamin Habegger --- .../org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java index 6ee23643ea0..ae899f26840 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -92,7 +92,7 @@ public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefin } newImageLuceneDefinitions.getChildren().put("/oak:index/" + diffIndex, diffObj); } catch (Exception e) { - String message = "Error parsing diff.index"; + String message = "Error parsing " + diffIndex; LOG.warn(message + ": {}", e.getMessage(), e); diffIndexDefinition.setProperty("error", message + ": " + e.getMessage()); } From 8ec6124e43fd8a9d1653610efdb419c51da1a513 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 2 Feb 2026 16:19:00 +0100 Subject: [PATCH 06/16] Update oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java Co-authored-by: Benjamin Habegger --- .../org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java index ae899f26840..38029156c29 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -93,7 +93,7 @@ public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefin newImageLuceneDefinitions.getChildren().put("/oak:index/" + diffIndex, diffObj); } catch (Exception e) { String message = "Error parsing " + diffIndex; - LOG.warn(message + ": {}", e.getMessage(), e); + LOG.warn("{}: {}", message, e.getMessage(), e); diffIndexDefinition.setProperty("error", message + ": " + e.getMessage()); } } From fbeab780e0d0f83d1d9b6b3238b7f9c24096230a Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 6 Feb 2026 15:25:11 +0100 Subject: [PATCH 07/16] Update oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java Co-authored-by: Amit Jain --- .../apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java index 38029156c29..f4dd1623d91 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -137,8 +137,7 @@ public static String tryReadString(PropertyState jcrData) { if (jcrData == null) { return null; } - InputStream in = jcrData.getValue(Type.BINARY).getNewStream(); - try { + try (InputStream in = jcrData.getValue(Type.BINARY).getNewStream()) { return new String(in.readAllBytes(), StandardCharsets.UTF_8); } catch (IOException e) { LOG.warn("Can not read jcr:data", e); From 2dfbe74c1db04ed611f3a626bac406f959eeb209 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 6 Feb 2026 15:26:54 +0100 Subject: [PATCH 08/16] Update oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java Co-authored-by: Amit Jain --- .../jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java index f6d1ebc9c4f..9a9b34b1566 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java @@ -435,7 +435,7 @@ public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject n merged.getProperties().remove("reindex"); if (!deleteCopiesOutOfTheBoxIndex && indexDiff.toString().equals("{}")) { merged.getProperties().put("type", "\"disabled\""); - merged.getProperties().put("mergeComment", "\"This index is superseeded and can be removed\""); + merged.getProperties().put("mergeComment", "\"This index is superseded and can be removed\""); } newImageLuceneDefinitions.getChildren().put(key, merged); return true; From f9e6fecffa96f6625fc3eb771b2d61be30a0d4a9 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 6 Feb 2026 15:27:28 +0100 Subject: [PATCH 09/16] Update oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java Co-authored-by: Amit Jain --- .../org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java index f4dd1623d91..b4413fb4eea 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -201,7 +201,7 @@ private static void disableOrRemoveOldVersions(NodeBuilder definitions, String i } } for (String r : toRemove) { - LOG.info("Removing old index " + r); + LOG.info("Removing old index {}", r); definitions.child(r).remove(); updateNodetypeIndexForPath(definitions, r, false); } From fcca18500f2f10f96bdb832da2736b12f77736ca Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 6 Feb 2026 15:29:14 +0100 Subject: [PATCH 10/16] Update oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java Co-authored-by: Benjamin Habegger --- .../jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java index 2f290748a97..da34188b7c4 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java @@ -74,7 +74,7 @@ public class JsonNodeBuilder { */ public static void addOrReplace(NodeBuilder builder, NodeStore nodeStore, String targetPath, String nodeType, String jsonString) throws CommitFailedException, IOException { LOG.info("Storing {}: {}", targetPath, jsonString); - if (nodeType.indexOf("/") >= 0) { + if (nodeType.contains("/")) { throw new IllegalStateException("Illegal node type: " + nodeType); } JsonObject json = JsonObject.fromJson(jsonString, true); From 0f71ddd61fb4e620f25e6e392c11e4c5fe5b7a85 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 6 Feb 2026 15:29:32 +0100 Subject: [PATCH 11/16] Update oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java Co-authored-by: Amit Jain --- .../org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java index b4413fb4eea..f6a1065ebe0 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -103,7 +103,7 @@ public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefin } LOG.info("Processing a new diff.index with node store {}", store); JsonObject repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(indexDefinitions); - LOG.debug("Index list {}", repositoryDefinitions.toString()); + LOG.debug("Index list {}", repositoryDefinitions); try { DiffIndexMerger.instance().merge(newImageLuceneDefinitions, repositoryDefinitions, store); for (String indexPath : newImageLuceneDefinitions.getChildren().keySet()) { From aee4d171e9d105ff9f2a82c14df18834ed565e35 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 6 Feb 2026 15:32:28 +0100 Subject: [PATCH 12/16] OAK-12010 Simplified index management --- .../oak/plugins/index/IndexName.java | 2 +- .../oak/plugins/index/diff/DiffIndex.java | 74 +++++++++++++------ .../plugins/index/diff/DiffIndexMerger.java | 40 +++++++--- .../oak/plugins/index/diff/DiffIndexTest.java | 2 +- .../oak/plugins/index/diff/MergeTest.java | 13 ++-- 5 files changed, 93 insertions(+), 38 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java index 7d8313c8e22..994fe62a79a 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java @@ -234,7 +234,7 @@ public static Collection filterReplacedIndexes(Collection indexP return result; } - public static Collection filterNewestIndexes(Collection indexPaths, NodeState rootState) { + public static Collection filterNewestIndexes(Collection indexPaths) { HashMap latestVersions = new HashMap<>(); for (String p : indexPaths) { IndexName indexName = IndexName.parse(p); diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java index f6a1065ebe0..161558320a6 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -30,6 +30,7 @@ import org.apache.jackrabbit.oak.plugins.index.IndexConstants; import org.apache.jackrabbit.oak.plugins.index.IndexName; import org.apache.jackrabbit.oak.plugins.tree.TreeConstants; +import org.apache.jackrabbit.oak.spi.nodetype.NodeTypeConstants; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.slf4j.Logger; @@ -46,6 +47,8 @@ public class DiffIndex { private static final Logger LOG = LoggerFactory.getLogger(DiffIndex.class); + private final static DiffIndexMerger MERGER = new DiffIndexMerger(); + /** * Apply changes to the index definitions. That means merge the index diff with * the existing indexes, creating new index versions. It might also mean to @@ -55,8 +58,25 @@ public class DiffIndex { * @param indexDefinitions the /oak:index node */ public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefinitions) { - JsonObject newImageLuceneDefinitions = null; - for (String diffIndex : new String[] { DiffIndexMerger.DIFF_INDEX, DiffIndexMerger.DIFF_INDEX_OPTIMIZER }) { + JsonObject diffs = collectDiffs(indexDefinitions); + if (diffs == null) { + // nothing todo + return; + } + processDiffs(store, indexDefinitions, diffs); + } + + /** + * Collect the diffs from the diff.index and diff.index.optimizer. + * + * @param indexDefinitions the node builder for /oak:index + * @return the diffs, or null if none + */ + public static JsonObject collectDiffs(NodeBuilder indexDefinitions) { + JsonObject diffs = null; + for (String diffIndex : new String[] { + DiffIndexMerger.DIFF_INDEX, + DiffIndexMerger.DIFF_INDEX_OPTIMIZER }) { if (!indexDefinitions.hasChildNode(diffIndex)) { continue; } @@ -65,12 +85,12 @@ public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefin if (!diffContent.exists()) { continue; } - PropertyState lastMod = diffContent.getProperty("jcr:lastModified"); + PropertyState lastMod = diffContent.getProperty(NodeTypeConstants.JCR_LASTMODIFIED); if (lastMod == null) { continue; } String modified = lastMod.getValue(Type.DATE); - PropertyState lastProcessed = diffContent.getProperty(":lastProcessed"); + PropertyState lastProcessed = diffContent.getProperty(DiffIndexMerger.LAST_PROCESSED); if (lastProcessed != null) { if (modified.equals(lastProcessed.getValue(Type.STRING))) { // already processed @@ -78,7 +98,7 @@ public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefin } } // store now, so a change is only processed once - diffContent.setProperty(":lastProcessed", modified); + diffContent.setProperty(DiffIndexMerger.LAST_PROCESSED, modified); PropertyState jcrData = diffContent.getProperty("jcr:data"); String diff = tryReadString(jcrData); if (diff == null) { @@ -87,39 +107,47 @@ public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefin try { JsonObject diffObj = JsonObject.fromJson("{\"diff\": " + diff + "}", true); diffIndexDefinition.removeProperty("error"); - if (newImageLuceneDefinitions == null) { - newImageLuceneDefinitions = new JsonObject(); + if (diffs == null) { + diffs = new JsonObject(); } - newImageLuceneDefinitions.getChildren().put("/oak:index/" + diffIndex, diffObj); + diffs.getChildren().put("/oak:index/" + diffIndex, diffObj); } catch (Exception e) { String message = "Error parsing " + diffIndex; LOG.warn("{}: {}", message, e.getMessage(), e); diffIndexDefinition.setProperty("error", message + ": " + e.getMessage()); } } - if (newImageLuceneDefinitions == null) { - // not a valid diff index, or already processed - return; - } - LOG.info("Processing a new diff.index with node store {}", store); + return diffs; + } + + /** + * Process the diffs. + * + * @param store the node store + * @param indexDefinitions the node builder for /oak:index + * @param diffs the json object with the combined diffs + */ + private static void processDiffs(NodeStore store, NodeBuilder indexDefinitions, JsonObject diffs) { + LOG.info("Processing a diffs"); JsonObject repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(indexDefinitions); LOG.debug("Index list {}", repositoryDefinitions); try { - DiffIndexMerger.instance().merge(newImageLuceneDefinitions, repositoryDefinitions, store); - for (String indexPath : newImageLuceneDefinitions.getChildren().keySet()) { + MERGER.merge(diffs, repositoryDefinitions, store); + for (String indexPath : diffs.getChildren().keySet()) { if (indexPath.startsWith("/oak:index/" + DiffIndexMerger.DIFF_INDEX)) { continue; } - JsonObject newDef = newImageLuceneDefinitions.getChildren().get(indexPath); + JsonObject newDef = diffs.getChildren().get(indexPath); String indexName = PathUtils.getName(indexPath); - JsonNodeBuilder.addOrReplace(indexDefinitions, store, indexName, IndexConstants.INDEX_DEFINITIONS_NODE_TYPE, newDef.toString()); + JsonNodeBuilder.addOrReplace(indexDefinitions, store, indexName, + IndexConstants.INDEX_DEFINITIONS_NODE_TYPE, newDef.toString()); updateNodetypeIndexForPath(indexDefinitions, indexName, true); disableOrRemoveOldVersions(indexDefinitions, indexPath, indexName); } removeDisabledMergedIndexes(indexDefinitions); sortIndexes(indexDefinitions); } catch (Exception e) { - LOG.warn("Error merging diff.index: {}", e.getMessage(), e); + LOG.warn("Error merging diffs: {}", e.getMessage(), e); NodeBuilder diffIndexDefinition = indexDefinitions.child(DiffIndexMerger.DIFF_INDEX); diffIndexDefinition.setProperty("error", e.getMessage()); } @@ -157,10 +185,12 @@ private static void sortIndexes(NodeBuilder builder) { private static void removeDisabledMergedIndexes(NodeBuilder definitions) { ArrayList toRemove = new ArrayList<>(); for (String child : definitions.getChildNodeNames()) { - if (!definitions.getChildNode(child).hasProperty("mergeChecksum")) { + if (!definitions.getChildNode(child).hasProperty(DiffIndexMerger.MERGE_CHECKSUM)) { continue; } - if ("disabled".equals(definitions.getChildNode(child).getString("type"))) { + if (IndexConstants.TYPE_DISABLED.equals(definitions. + getChildNode(child). + getString(IndexConstants.TYPE_PROPERTY_NAME))) { toRemove.add(child); } } @@ -193,7 +223,9 @@ private static void disableOrRemoveOldVersions(NodeBuilder definitions, String i String childBaseName = IndexName.parse(child).getBaseName(); if (baseName.equals(childBaseName)) { if (indexName.equals(child)) { - if (!"disabled".equals(definitions.getChildNode(indexName).getString("type"))) { + if (!IndexConstants.TYPE_DISABLED.equals(definitions. + getChildNode(indexName). + getString(IndexConstants.TYPE_PROPERTY_NAME))) { continue; } } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java index 9a9b34b1566..07c0d872e9d 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java @@ -50,6 +50,9 @@ public class DiffIndexMerger { public final static String DIFF_INDEX = "diff.index"; public final static String DIFF_INDEX_OPTIMIZER = "diff.index.optimizer"; + public final static String LAST_PROCESSED = ":lastProcessed"; + public final static String MERGE_CHECKSUM = "mergeChecksum"; + private final static String MERGE_INFO = "This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html"; // the list of unsupported included paths, e.g. "/apps,/libs" @@ -65,16 +68,13 @@ public class DiffIndexMerger { // whether to log at info level private final static boolean LOG_AT_INFO_LEVEL = Boolean.getBoolean("oak.diffIndex.logAtInfoLevel"); - private final String[] unsupportedIncludedPaths; - private final boolean deleteCreatesDummyIndex; - private final boolean deleteCopiesOutOfTheBoxIndex; - private final boolean logAtInfoLevel; - - static final DiffIndexMerger INSTANCE = new DiffIndexMerger(UNSUPPORTED_INCLUDED_PATHS, - DELETE_CREATES_DUMMY, DELETE_COPIES_OOTB, LOG_AT_INFO_LEVEL); + private String[] unsupportedIncludedPaths; + private boolean deleteCreatesDummyIndex; + private boolean deleteCopiesOutOfTheBoxIndex; + private boolean logAtInfoLevel; - public static DiffIndexMerger instance() { - return INSTANCE; + public DiffIndexMerger() { + this(UNSUPPORTED_INCLUDED_PATHS, DELETE_CREATES_DUMMY, DELETE_COPIES_OOTB, LOG_AT_INFO_LEVEL); } DiffIndexMerger(String[] unsupportedIncludedPaths, @@ -495,7 +495,7 @@ private static String computeMergeChecksum(JsonObject json) { return StringUtils.convertBytesToHex(md.digest(bytes)); } catch (NoSuchAlgorithmException e) { // SHA-256 is guaranteed to be available in standard Java platforms - throw new RuntimeException("SHA-256 algorithm not available", e); + throw new IllegalStateException("SHA-256 algorithm not available", e); } } @@ -830,4 +830,24 @@ private void log(String format, Object... arguments) { } } + public DiffIndexMerger setUnsupportedIncludedPaths(String[] unsupportedIncludedPaths) { + this.unsupportedIncludedPaths = unsupportedIncludedPaths; + return this; + } + + public DiffIndexMerger setDeleteCreatesDummyIndex(boolean deleteCreatesDummyIndex) { + this.deleteCreatesDummyIndex = deleteCreatesDummyIndex; + return this; + } + + public DiffIndexMerger setDeleteCopiesOutOfTheBoxIndex(boolean deleteCopiesOutOfTheBoxIndex) { + this.deleteCopiesOutOfTheBoxIndex = deleteCopiesOutOfTheBoxIndex; + return this; + } + + public DiffIndexMerger setLogAtInfoLevel(boolean logAtInfoLevel) { + this.logAtInfoLevel = logAtInfoLevel; + return this; + } + } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java index e6a5823599e..b0cf3b50a1b 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexTest.java @@ -260,7 +260,7 @@ public void testDiffIndexUpdate() throws Exception { private void assertSameJson(String a, String b) { JsonObject ja = JsonObject.fromJson(a, true); JsonObject jb = JsonObject.fromJson(b, true); - if (!DiffIndexMerger.instance().isSameIgnorePropertyOrder(ja, jb)) { + if (!new DiffIndexMerger().isSameIgnorePropertyOrder(ja, jb)) { assertEquals(a, b); } } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java index c55d1c3ab6c..b8e63fe9c3e 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/MergeTest.java @@ -110,7 +110,7 @@ public void renamedProperty() { // A property might be indexed twice, by adding two children to the "properties" node // that both have the same "name" value. // Alternatively, they could have the same "function" value. - String merged = DiffIndexMerger.instance().processMerge(JsonObject.fromJson("{\n" + String merged = new DiffIndexMerger().processMerge(JsonObject.fromJson("{\n" + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + " \"type\": \"lucene\",\n" + " \"indexRules\": {\n" @@ -164,7 +164,7 @@ public void renamedProperty() { public void renamedFunction() { // A function might be indexed twice, by adding two children to the "properties" node // that both have the same "function" value. - String merged = DiffIndexMerger.instance().processMerge(JsonObject.fromJson("{\n" + String merged = new DiffIndexMerger().processMerge(JsonObject.fromJson("{\n" + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + " \"type\": \"lucene\",\n" + " \"indexRules\": {\n" @@ -245,7 +245,7 @@ public void boost() { // - "analyzed" must not be overwritten // - "ordered" is added // - "boost" is overwritten - String merged = DiffIndexMerger.instance().processMerge(JsonObject.fromJson("{\n" + String merged = new DiffIndexMerger().processMerge(JsonObject.fromJson("{\n" + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + " \"type\": \"lucene\",\n" + " \"indexRules\": {\n" @@ -387,7 +387,10 @@ public void switchToLuceneChildrenTest() { @Test public void includesUnsupportedPathsTest() { - DiffIndexMerger merger = new DiffIndexMerger(new String[]{"/apps", "/libs"}, false, false, false); + DiffIndexMerger merger = new DiffIndexMerger(). + setUnsupportedIncludedPaths(new String[]{"/apps", "/libs"}). + setDeleteCopiesOutOfTheBoxIndex(false). + setDeleteCreatesDummyIndex(false); assertEquals(true, merger.includesUnsupportedPaths(null)); assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/"})); @@ -424,7 +427,7 @@ public void readDiffIndexTest() throws CommitFailedException { testProp.setProperty("propertyIndex", true); store.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY); - Map result = DiffIndexMerger.instance().readDiffIndex(store, "diff.index.optimizer"); + Map result = new DiffIndexMerger().readDiffIndex(store, "diff.index.optimizer"); assertEquals(1, result.size()); assertTrue(result.containsKey("/oak:index/diff.index.optimizer")); From 7097f768991bcba06e0d4a282ea542ac7766ae07 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 6 Feb 2026 15:44:00 +0100 Subject: [PATCH 13/16] OAK-12010 Simplified index management --- .../oak/plugins/index/search/spi/query/FulltextIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java index 0d47a4d148b..61ac6c6e4df 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java @@ -119,7 +119,7 @@ public List getPlans(Filter filter, List sortOrder, NodeS if (filterReplacedIndexes()) { indexPaths = IndexName.filterReplacedIndexes(indexPaths, rootState, runIsActiveIndexCheck()); } else { - indexPaths = IndexName.filterNewestIndexes(indexPaths, rootState); + indexPaths = IndexName.filterNewestIndexes(indexPaths); } List plans = new ArrayList<>(indexPaths.size()); for (String path : indexPaths) { From d954a1fd32fbeaed5da70c235a38d2a836fe7add Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 6 Feb 2026 16:21:41 +0100 Subject: [PATCH 14/16] OAK-12010 Simplified index management --- .../plugins/index/search/spi/query/IndexNameTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexNameTest.java b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexNameTest.java index cdcd14fe1bc..3498651d9d5 100644 --- a/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexNameTest.java +++ b/oak-search/src/test/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/IndexNameTest.java @@ -119,11 +119,10 @@ public void recursiveActive() { @Test public void filterNewestIndexes() { - NodeState root = EMPTY_NODE; // Single index - should return as-is Collection single = Arrays.asList("/lucene"); - Collection result = IndexName.filterNewestIndexes(single, root); + Collection result = IndexName.filterNewestIndexes(single); assertEquals(1, result.size()); assertTrue(result.contains("/lucene")); @@ -135,7 +134,7 @@ public void filterNewestIndexes() { "/lucene-1-custom-1", "/lucene-2-custom-3" ); - result = IndexName.filterNewestIndexes(multipleVersions, root); + result = IndexName.filterNewestIndexes(multipleVersions); assertEquals(1, result.size()); assertTrue(result.contains("/lucene-2-custom-3")); @@ -147,7 +146,7 @@ public void filterNewestIndexes() { "/luceneB-2-custom-1", "/luceneC-1-custom-5" ); - result = IndexName.filterNewestIndexes(differentBases, root); + result = IndexName.filterNewestIndexes(differentBases); assertEquals(new HashSet<>(Arrays.asList("/luceneA-1", "/luceneB-2-custom-1", "/luceneC-1-custom-5")), new HashSet<>(result)); @@ -157,13 +156,13 @@ public void filterNewestIndexes() { "/lucene-custom-2", "/lucene-custom-3" ); - result = IndexName.filterNewestIndexes(customOnly, root); + result = IndexName.filterNewestIndexes(customOnly); assertEquals(1, result.size()); assertTrue(result.contains("/lucene-custom-3")); // Empty collection Collection empty = Arrays.asList(); - result = IndexName.filterNewestIndexes(empty, root); + result = IndexName.filterNewestIndexes(empty); assertTrue(result.isEmpty()); } } From 2f31a0b95068d5da56a066d64ec48a0161bae65c Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 9 Feb 2026 09:28:11 +0100 Subject: [PATCH 15/16] OAK-12010 Simplified index management --- oak-core/DiffIndex.java | 242 +++++ oak-core/DiffIndexMerger.java | 833 ++++++++++++++++++ oak-core/DiffIndexTest.java | 362 ++++++++ oak-core/MergeTest.java | 437 +++++++++ .../oak/plugins/index/diff/DiffIndex.java | 2 +- .../plugins/index/diff/DiffIndexMerger.java | 16 +- ...nNodeBuilder.java => JsonNodeUpdater.java} | 19 +- .../index/diff/JsonNodeBuilderTest.java | 60 +- 8 files changed, 1923 insertions(+), 48 deletions(-) create mode 100644 oak-core/DiffIndex.java create mode 100644 oak-core/DiffIndexMerger.java create mode 100644 oak-core/DiffIndexTest.java create mode 100644 oak-core/MergeTest.java rename oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/{JsonNodeBuilder.java => JsonNodeUpdater.java} (95%) diff --git a/oak-core/DiffIndex.java b/oak-core/DiffIndex.java new file mode 100644 index 00000000000..6ee23643ea0 --- /dev/null +++ b/oak-core/DiffIndex.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexName; +import org.apache.jackrabbit.oak.plugins.tree.TreeConstants; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Processing of diff indexes, that is nodes under "/oak:index/diff.index". A + * diff index contains differences to existing indexes, and possibly new + * (custom) indexes in the form of JSON. These changes can then be merged + * (applied) to the index definitions. This allows to simplify index management, + * because it allows to modify (add, update) indexes in a simple way. + */ +public class DiffIndex { + + private static final Logger LOG = LoggerFactory.getLogger(DiffIndex.class); + + /** + * Apply changes to the index definitions. That means merge the index diff with + * the existing indexes, creating new index versions. It might also mean to + * remove old (merged) indexes if the diff no longer contains them. + * + * @param store the node store + * @param indexDefinitions the /oak:index node + */ + public static void applyDiffIndexChanges(NodeStore store, NodeBuilder indexDefinitions) { + JsonObject newImageLuceneDefinitions = null; + for (String diffIndex : new String[] { DiffIndexMerger.DIFF_INDEX, DiffIndexMerger.DIFF_INDEX_OPTIMIZER }) { + if (!indexDefinitions.hasChildNode(diffIndex)) { + continue; + } + NodeBuilder diffIndexDefinition = indexDefinitions.child(diffIndex); + NodeBuilder diffContent = diffIndexDefinition.getChildNode("diff.json").getChildNode("jcr:content"); + if (!diffContent.exists()) { + continue; + } + PropertyState lastMod = diffContent.getProperty("jcr:lastModified"); + if (lastMod == null) { + continue; + } + String modified = lastMod.getValue(Type.DATE); + PropertyState lastProcessed = diffContent.getProperty(":lastProcessed"); + if (lastProcessed != null) { + if (modified.equals(lastProcessed.getValue(Type.STRING))) { + // already processed + continue; + } + } + // store now, so a change is only processed once + diffContent.setProperty(":lastProcessed", modified); + PropertyState jcrData = diffContent.getProperty("jcr:data"); + String diff = tryReadString(jcrData); + if (diff == null) { + continue; + } + try { + JsonObject diffObj = JsonObject.fromJson("{\"diff\": " + diff + "}", true); + diffIndexDefinition.removeProperty("error"); + if (newImageLuceneDefinitions == null) { + newImageLuceneDefinitions = new JsonObject(); + } + newImageLuceneDefinitions.getChildren().put("/oak:index/" + diffIndex, diffObj); + } catch (Exception e) { + String message = "Error parsing diff.index"; + LOG.warn(message + ": {}", e.getMessage(), e); + diffIndexDefinition.setProperty("error", message + ": " + e.getMessage()); + } + } + if (newImageLuceneDefinitions == null) { + // not a valid diff index, or already processed + return; + } + LOG.info("Processing a new diff.index with node store {}", store); + JsonObject repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(indexDefinitions); + LOG.debug("Index list {}", repositoryDefinitions.toString()); + try { + DiffIndexMerger.instance().merge(newImageLuceneDefinitions, repositoryDefinitions, store); + for (String indexPath : newImageLuceneDefinitions.getChildren().keySet()) { + if (indexPath.startsWith("/oak:index/" + DiffIndexMerger.DIFF_INDEX)) { + continue; + } + JsonObject newDef = newImageLuceneDefinitions.getChildren().get(indexPath); + String indexName = PathUtils.getName(indexPath); + JsonNodeBuilder.addOrReplace(indexDefinitions, store, indexName, IndexConstants.INDEX_DEFINITIONS_NODE_TYPE, newDef.toString()); + updateNodetypeIndexForPath(indexDefinitions, indexName, true); + disableOrRemoveOldVersions(indexDefinitions, indexPath, indexName); + } + removeDisabledMergedIndexes(indexDefinitions); + sortIndexes(indexDefinitions); + } catch (Exception e) { + LOG.warn("Error merging diff.index: {}", e.getMessage(), e); + NodeBuilder diffIndexDefinition = indexDefinitions.child(DiffIndexMerger.DIFF_INDEX); + diffIndexDefinition.setProperty("error", e.getMessage()); + } + } + + /** + * Try to read a text from the (binary) jcr:data property. Edge cases such as + * "property does not exist" and IO exceptions (blob not found) do not throw an + * exception (IO exceptions are logged). + * + * @param jcrData the "jcr:data" property + * @return the string, or null if reading fails + */ + public static String tryReadString(PropertyState jcrData) { + if (jcrData == null) { + return null; + } + InputStream in = jcrData.getValue(Type.BINARY).getNewStream(); + try { + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + LOG.warn("Can not read jcr:data", e); + return null; + } + } + + private static void sortIndexes(NodeBuilder builder) { + ArrayList list = new ArrayList<>(); + for (String child : builder.getChildNodeNames()) { + list.add(child); + } + list.sort(Comparator.naturalOrder()); + builder.setProperty(TreeConstants.OAK_CHILD_ORDER, list, Type.NAMES); + } + + private static void removeDisabledMergedIndexes(NodeBuilder definitions) { + ArrayList toRemove = new ArrayList<>(); + for (String child : definitions.getChildNodeNames()) { + if (!definitions.getChildNode(child).hasProperty("mergeChecksum")) { + continue; + } + if ("disabled".equals(definitions.getChildNode(child).getString("type"))) { + toRemove.add(child); + } + } + for (String r : toRemove) { + LOG.info("Removing disabled index {}", r); + definitions.child(r).remove(); + updateNodetypeIndexForPath(definitions, r, false); + } + } + + /** + * Try to remove or disable old version of merged indexes, if there are any. + * + * @param definitions the builder for /oak:index + * @param indexPath the path + * @param keep which index name (which version) to retain + */ + private static void disableOrRemoveOldVersions(NodeBuilder definitions, String indexPath, String keep) { + String indexName = indexPath; + if (indexPath.startsWith("/oak:index/")) { + indexName = indexPath.substring("/oak:index/".length()); + } + String baseName = IndexName.parse(indexName).getBaseName(); + ArrayList toRemove = new ArrayList<>(); + for (String child : definitions.getChildNodeNames()) { + if (child.equals(keep) || child.indexOf("-custom-") < 0) { + // the one to keep, or not a customized or custom index + continue; + } + String childBaseName = IndexName.parse(child).getBaseName(); + if (baseName.equals(childBaseName)) { + if (indexName.equals(child)) { + if (!"disabled".equals(definitions.getChildNode(indexName).getString("type"))) { + continue; + } + } + toRemove.add(child); + } + } + for (String r : toRemove) { + LOG.info("Removing old index " + r); + definitions.child(r).remove(); + updateNodetypeIndexForPath(definitions, r, false); + } + } + + private static void updateNodetypeIndexForPath(NodeBuilder indexDefinitions, + String indexName, boolean add) { + LOG.info("nodetype index update add={} name={}", add, indexName); + if (!indexDefinitions.hasChildNode("nodetype")) { + return; + } + NodeBuilder nodetypeIndex = indexDefinitions.getChildNode("nodetype"); + NodeBuilder indexContent = nodetypeIndex.child(":index"); + String key = URLEncoder.encode("oak:QueryIndexDefinition", StandardCharsets.UTF_8); + String path = "/oak:index/" + indexName; + if (add) { + // insert entry + NodeBuilder builder = indexContent.child(key); + for (String name : PathUtils.elements(path)) { + builder = builder.child(name); + } + LOG.info("nodetype index match"); + builder.setProperty("match", true); + } else { + // remove entry (for deleted indexes) + NodeBuilder builder = indexContent.getChildNode(key); + for (String name : PathUtils.elements(path)) { + builder = builder.getChildNode(name); + } + if (builder.exists()) { + LOG.info("nodetype index remove"); + builder.removeProperty("match"); + } + } + } + +} diff --git a/oak-core/DiffIndexMerger.java b/oak-core/DiffIndexMerger.java new file mode 100644 index 00000000000..f6d1ebc9c4f --- /dev/null +++ b/oak-core/DiffIndexMerger.java @@ -0,0 +1,833 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.jackrabbit.oak.commons.StringUtils; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.commons.json.JsopBuilder; +import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; +import org.apache.jackrabbit.oak.json.Base64BlobSerializer; +import org.apache.jackrabbit.oak.json.JsonSerializer; +import org.apache.jackrabbit.oak.plugins.index.IndexName; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Index definition merge utility that uses the "diff" mode. + */ +public class DiffIndexMerger { + + private final static Logger LOG = LoggerFactory.getLogger(DiffIndexMerger.class); + + public final static String DIFF_INDEX = "diff.index"; + public final static String DIFF_INDEX_OPTIMIZER = "diff.index.optimizer"; + + private final static String MERGE_INFO = "This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html"; + + // the list of unsupported included paths, e.g. "/apps,/libs" + // by default all paths are supported + private final static String[] UNSUPPORTED_INCLUDED_PATHS = System.getProperty("oak.diffIndex.unsupportedPaths", "").split(","); + + // in case a custom index is removed, whether a dummy index is created + private final static boolean DELETE_CREATES_DUMMY = Boolean.getBoolean("oak.diffIndex.deleteCreatesDummy"); + + // in case a customization was removed, create a copy of the OOTB index + private final static boolean DELETE_COPIES_OOTB = Boolean.getBoolean("oak.diffIndex.deleteCopiesOOTB"); + + // whether to log at info level + private final static boolean LOG_AT_INFO_LEVEL = Boolean.getBoolean("oak.diffIndex.logAtInfoLevel"); + + private final String[] unsupportedIncludedPaths; + private final boolean deleteCreatesDummyIndex; + private final boolean deleteCopiesOutOfTheBoxIndex; + private final boolean logAtInfoLevel; + + static final DiffIndexMerger INSTANCE = new DiffIndexMerger(UNSUPPORTED_INCLUDED_PATHS, + DELETE_CREATES_DUMMY, DELETE_COPIES_OOTB, LOG_AT_INFO_LEVEL); + + public static DiffIndexMerger instance() { + return INSTANCE; + } + + DiffIndexMerger(String[] unsupportedIncludedPaths, + boolean deleteCreatesDummyIndex, boolean deleteCopiesOutOfTheBoxIndex, + boolean logAtInfoLevel) { + this.unsupportedIncludedPaths = unsupportedIncludedPaths; + this.deleteCreatesDummyIndex = deleteCreatesDummyIndex; + this.deleteCopiesOutOfTheBoxIndex = deleteCopiesOutOfTheBoxIndex; + this.logAtInfoLevel = logAtInfoLevel; + } + + /** + * If there is a diff index, that is an index with prefix "diff.", then try to merge it. + * + * @param newImageLuceneDefinitions + * the new indexes + * (input and output) + * @param repositoryDefinitions + * the indexes in the writable repository + * (input) + * @param repositoryNodeStore + */ + public void merge(JsonObject newImageLuceneDefinitions, JsonObject repositoryDefinitions, NodeStore repositoryNodeStore) { + // combine all definitions into one object + JsonObject combined = new JsonObject(); + + // index definitions in the repository + combined.getChildren().putAll(repositoryDefinitions.getChildren()); + + // read the diff.index.optimizer explicitly, + // because it's a not a regular index definition, + // and so in the repositoryDefinitions + if (repositoryNodeStore != null) { + Map diffInRepo = readDiffIndex(repositoryNodeStore, DIFF_INDEX_OPTIMIZER); + combined.getChildren().putAll(diffInRepo); + } + + // overwrite with the provided definitions (if any) + combined.getChildren().putAll(newImageLuceneDefinitions.getChildren()); + + // check if there "diff.index" or "diff.index.optimizer" + boolean found = combined.getChildren().containsKey("/oak:index/" + DIFF_INDEX) + || combined.getChildren().containsKey("/oak:index/" + DIFF_INDEX_OPTIMIZER); + if (!found) { + // early exit, so that the risk of merging the PR + // is very small for customers that do not use this + log("No 'diff.index' definition"); + return; + } + mergeDiff(newImageLuceneDefinitions, combined); + } + + /** + * If there is a diff index (hardcoded node "/oak:index/diff.index" or + * "/oak:index/diff.index.optimizer"), then iterate over all entries and create new + * (merged) versions if needed. + * + * @param newImageLuceneDefinitions + * the new Lucene definitions + * (input + output) + * @param combined + * the definitions in the repository, + * including the one in the customer repo and new ones + * (input) + * @return whether a new version of an index was added + */ + boolean mergeDiff(JsonObject newImageLuceneDefinitions, JsonObject combined) { + // iterate again, this time process + + // collect the diff index(es) + HashMap toProcess = new HashMap<>(); + tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX, toProcess); + tryExtractDiffIndex(combined, "/oak:index/" + DIFF_INDEX_OPTIMIZER, toProcess); + // if the diff index exists, but doesn't contain some of the previous indexes + // (indexes with mergeInfo), then we need to disable those (using /dummy includedPath) + extractExistingMergedIndexes(combined, toProcess); + if (toProcess.isEmpty()) { + log("No diff index definitions found."); + return false; + } + boolean hasChanges = false; + for (Entry e : toProcess.entrySet()) { + String key = e.getKey(); + JsonObject value = e.getValue(); + if (key.startsWith("/oak:index/")) { + LOG.warn("The key should contains just the index name, without the '/oak:index' prefix for key {}", key); + key = key.substring("/oak:index/".length()); + } + log("Processing {}", key); + hasChanges |= processMerge(key, value, newImageLuceneDefinitions, combined); + } + return hasChanges; + } + + /** + * Extract a "diff.index" from the set of index definitions (if found), and if + * found, store the nested entries in the target map, merging them with previous + * entries if found. + * + * The diff.index may either have a file (a "jcr:content" child node with a + * "jcr:data" property), or a "diff" JSON object. For customers (in the git + * repository), the file is much easier to construct, but when running the + * indexing job, the nested JSON is much easier. + * + * @param indexDefs the set of index definitions (may be empty) + * @param name the name of the diff.index (either diff.index or + * diff.index.optimizer) + * @param target the target map of diff.index definitions + * @return the error message trying to parse the JSON file, or null + */ + public static String tryExtractDiffIndex(JsonObject indexDefs, String name, HashMap target) { + JsonObject diffIndex = indexDefs.getChildren().get(name); + if (diffIndex == null) { + return null; + } + // extract either the file, or the nested json + JsonObject file = diffIndex.getChildren().get("diff.json"); + JsonObject diff; + if (file != null) { + // file + JsonObject jcrContent = file.getChildren().get("jcr:content"); + if (jcrContent == null) { + String message = "jcr:content child node is missing in diff.json"; + LOG.warn(message); + return message; + } + String jcrData = JsonNodeBuilder.oakStringValue(jcrContent, "jcr:data"); + try { + diff = JsonObject.fromJson(jcrData, true); + } catch (Exception e) { + LOG.warn("Illegal Json, ignoring: {}", jcrData, e); + String message = "Illegal Json, ignoring: " + e.getMessage(); + return message; + } + } else { + // nested json + diff = diffIndex.getChildren().get("diff"); + } + // store, if not empty + if (diff != null) { + for (Entry e : diff.getChildren().entrySet()) { + String key = e.getKey(); + target.put(key, mergeDiffs(target.get(key), e.getValue())); + } + } + return null; + } + + /** + * Extract the indexes with a "mergeInfo" property and store them in the target + * object. This is needed so that indexes that were removed from the index.diff + * are detected (a new version is needed in this case with includedPaths + * "/dummy"). + * + * @param indexDefs the index definitions in the repository + * @param target the target map of "diff.index" definitions. for each entry + * found, an empty object is added + */ + private static void extractExistingMergedIndexes(JsonObject indexDefs, HashMap target) { + for (Entry e : indexDefs.getChildren().entrySet()) { + String key = e.getKey(); + JsonObject value = e.getValue(); + if (key.indexOf("-custom-") < 0 || !value.getProperties().containsKey("mergeInfo")) { + continue; + } + String baseName = IndexName.parse(key.substring("/oak:index/".length())).getBaseName(); + if (!target.containsKey(baseName)) { + // if there is no entry yet for this key, + // add a new empty object + target.put(baseName, new JsonObject()); + } + } + } + + /** + * Merge diff from "diff.index" and "diff.index.optimizer". + * The customer can define a diff (stored in "diff.index") + * and someone else (or the optimizer) can define one (stored in "diff.index.optimizer"). + * + * @param a the first diff + * @param b the second diff (overwrites entries in a) + * @return the merged entry + */ + public static JsonObject mergeDiffs(JsonObject a, JsonObject b) { + if (a == null) { + return b; + } else if (b == null) { + return a; + } + JsonObject result = JsonObject.fromJson(a.toString(), true); + result.getProperties().putAll(b.getProperties()); + HashSet both = new HashSet<>(a.getChildren().keySet()); + both.addAll(b.getChildren().keySet()); + for (String k : both) { + result.getChildren().put(k, mergeDiffs(a.getChildren().get(k), b.getChildren().get(k))); + } + return result; + } + + /** + * Merge using the diff definition. + * + * If the latest customized index already matches, then + * newImageLuceneDefinitions will remain as is. Otherwise, a new customized + * index is added, with a "mergeInfo" property. + * + * Existing properties are never changed; only new properties/children are + * added. + * + * @param indexName the name, eg. "damAssetLucene" + * @param indexDiff the diff with the new properties + * @param newImageLuceneDefinitions the new Lucene definitions (input + output) + * @param combined the definitions in the repository, including + * the one in the customer repo and new ones + * (input) + * @return whether a new version of an index was added + */ + public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject newImageLuceneDefinitions, JsonObject combined) { + // extract the latest product index (eg. damAssetLucene-12) + // and customized index (eg. damAssetLucene-12-custom-3) - if any + IndexName latestProduct = null; + String latestProductKey = null; + IndexName latestCustomized = null; + String latestCustomizedKey = null; + String prefix = "/oak:index/"; + for (String key : combined.getChildren().keySet()) { + IndexName name = IndexName.parse(key.substring(prefix.length())); + if (!name.isVersioned()) { + log("Ignoring unversioned index {}", name); + continue; + } + if (!name.getBaseName().equals(indexName)) { + continue; + } + boolean isCustom = key.indexOf("-custom-") >= 0; + if (isCustom) { + if (latestCustomized == null || + name.compareTo(latestCustomized) > 0) { + latestCustomized = name; + latestCustomizedKey = key; + } + } else { + if (latestProduct == null || + name.compareTo(latestProduct) > 0) { + latestProduct = name; + latestProductKey = key; + } + } + } + log("Latest product: {}", latestProductKey); + log("Latest customized: {}", latestCustomizedKey); + if (latestProduct == null) { + if (indexName.indexOf('.') >= 0) { + // a fully custom index needs to contains a dot + log("Fully custom index {}", indexName); + } else { + log("No product version for {}", indexName); + return false; + } + } + JsonObject latestProductIndex = combined.getChildren().get(latestProductKey); + String[] includedPaths; + if (latestProductIndex == null) { + if (indexDiff.getProperties().isEmpty() && indexDiff.getChildren().isEmpty()) { + // there is no customization (any more), which means a dummy index may be needed + log("No customization for {}", indexName); + } else { + includedPaths = JsonNodeBuilder.oakStringArrayValue(indexDiff, "includedPaths"); + if (includesUnsupportedPaths(includedPaths)) { + LOG.warn("New custom index {} is not supported because it contains an unsupported path ({})", + indexName, Arrays.toString(unsupportedIncludedPaths)); + return false; + } + } + } else { + includedPaths = JsonNodeBuilder.oakStringArrayValue(latestProductIndex, "includedPaths"); + if (includesUnsupportedPaths(includedPaths)) { + LOG.warn("Customizing index {} is not supported because it contains an unsupported path ({})", + latestProductKey, Arrays.toString(unsupportedIncludedPaths)); + return false; + } + } + + // merge + JsonObject merged = null; + if (indexDiff == null) { + // no diff definition: use to the OOTB index + if (latestCustomized == null) { + log("Only a product index found, nothing to do"); + return false; + } + merged = latestProductIndex; + } else { + merged = processMerge(latestProductIndex, indexDiff); + } + + // compare to the latest version of the this index + JsonObject latestIndexVersion = new JsonObject(); + if (latestCustomized == null) { + latestIndexVersion = latestProductIndex; + } else { + latestIndexVersion = combined.getChildren().get(latestCustomizedKey); + } + JsonObject mergedDef = cleanedAndNormalized(switchToLucene(merged)); + // compute merge checksum for later, but do not yet add + String mergeChecksum = computeMergeChecksum(mergedDef); + // get the merge checksum before cleaning (cleaning removes it) - if available + String key; + if (latestIndexVersion == null) { + // new index + key = prefix + indexName + "-1-custom-1"; + } else { + String latestMergeChecksum = JsonNodeBuilder.oakStringValue(latestIndexVersion, "mergeChecksum"); + JsonObject latestDef = cleanedAndNormalized(switchToLucene(latestIndexVersion)); + if (isSameIgnorePropertyOrder(mergedDef, latestDef)) { + // normal case: no change + // (even if checksums do not match: checksums might be missing or manipulated) + log("Latest index matches"); + if (latestMergeChecksum != null && !latestMergeChecksum.equals(mergeChecksum)) { + LOG.warn("Indexes do match, but checksums do not. Possibly checksum was changed: {} vs {}", latestMergeChecksum, mergeChecksum); + LOG.warn("latest: {}\nmerged: {}", latestDef, mergedDef); + } + return false; + } + if (latestMergeChecksum != null && latestMergeChecksum.equals(mergeChecksum)) { + // checksum matches, but data does not match + // could be eg. due to numbers formatting issues (-0.0 vs 0.0, 0.001 vs 1e-3) + // but unexpected because we do not normally have such cases + LOG.warn("Indexes do not match, but checksums match. Possible normalization issue."); + LOG.warn("Index: {}, latest: {}\nmerged: {}", indexName, latestDef, mergedDef); + // if checksums match, we consider it a match + return false; + } + LOG.info("Indexes do not match, with"); + LOG.info("Index: {}, latest: {}\nmerged: {}", indexName, latestDef, mergedDef); + // a new merged index definition + if (latestProduct == null) { + // fully custom index: increment version + key = prefix + indexName + + "-" + latestCustomized.getProductVersion() + + "-custom-" + (latestCustomized.getCustomerVersion() + 1); + } else { + // customized OOTB index: use the latest product as the base + key = prefix + indexName + + "-" + latestProduct.getProductVersion() + + "-custom-"; + if (latestCustomized != null) { + key += (latestCustomized.getCustomerVersion() + 1); + } else { + key += "1"; + } + } + } + merged.getProperties().put("mergeInfo", JsopBuilder.encode(MERGE_INFO)); + merged.getProperties().put("mergeChecksum", JsopBuilder.encode(mergeChecksum)); + merged.getProperties().put("merges", "[" + JsopBuilder.encode("/oak:index/" + indexName) + "]"); + merged.getProperties().remove("reindexCount"); + merged.getProperties().remove("reindex"); + if (!deleteCopiesOutOfTheBoxIndex && indexDiff.toString().equals("{}")) { + merged.getProperties().put("type", "\"disabled\""); + merged.getProperties().put("mergeComment", "\"This index is superseeded and can be removed\""); + } + newImageLuceneDefinitions.getChildren().put(key, merged); + return true; + } + + /** + * Check whether the includedPaths covers unsupported paths, + * if there are any unsupported path (eg. "/apps" or "/libs"). + * In this case, simplified index management is not supported. + * + * @param includedPaths the includedPaths list + * @return true if any unsupported path is included + */ + public boolean includesUnsupportedPaths(String[] includedPaths) { + if (unsupportedIncludedPaths.length == 1 && "".equals(unsupportedIncludedPaths[0])) { + // set to an empty string + return false; + } + if (includedPaths == null) { + // not set means all entries + return true; + } + for (String path : includedPaths) { + if ("/".equals(path)) { + // all + return true; + } + for (String unsupported : unsupportedIncludedPaths) { + if (unsupported.isEmpty()) { + continue; + } + if (path.equals(unsupported) || path.startsWith(unsupported + "/")) { + // includedPaths matches, or starts with an unsupported path + return true; + } + } + } + return false; + } + + /** + * Compute the SHA-256 checksum of the JSON object. This is useful to detect + * that the JSON object was not "significantly" changed, even if stored + * somewhere and later read again. Insignificant changes include: rounding of + * floating point numbers, re-ordering properties, things like that. Without the + * checksum, we would risk creating a new version of a customized index each + * time the indexing job is run, even thought the customer didn't change + * anything. + * + * @param json the input + * @return the SHA-256 checksum + */ + private static String computeMergeChecksum(JsonObject json) { + byte[] bytes = json.toString().getBytes(StandardCharsets.UTF_8); + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + return StringUtils.convertBytesToHex(md.digest(bytes)); + } catch (NoSuchAlgorithmException e) { + // SHA-256 is guaranteed to be available in standard Java platforms + throw new RuntimeException("SHA-256 algorithm not available", e); + } + } + + /** + * Switch the index from type "elasticsearch" to "lucene", if needed. This will + * also replace all properties that have an "...@lucene" version. + * + * This is needed because we want to merge only the "lucene" version, to + * simplify the merging logic. (The switch to the "elasticsearch" version + * happens later). + * + * @param indexDef the index definition (is not changed by this method) + * @return the lucene version (a new JSON object) + */ + public static JsonObject switchToLucene(JsonObject indexDef) { + JsonObject obj = JsonObject.fromJson(indexDef.toString(), true); + String type = JsonNodeBuilder.oakStringValue(obj, "type"); + if (type == null || !"elasticsearch".equals(type) ) { + return obj; + } + switchToLuceneChildren(obj); + return obj; + } + + public static void switchToLuceneChildren(JsonObject indexDef) { + // clone the keys to avoid ConcurrentModificationException + for (String p : new ArrayList<>(indexDef.getProperties().keySet())) { + if (!p.endsWith("@lucene")) { + continue; + } + String v = indexDef.getProperties().remove(p); + indexDef.getProperties().put(p.substring(0, p.length() - "@lucene".length()), v); + } + for (String c : indexDef.getChildren().keySet()) { + JsonObject co = indexDef.getChildren().get(c); + switchToLuceneChildren(co); + } + } + + /** + * Convert the JSON object to a new object, where index definition + * properties that are unimportant for comparison are removed. + * Example of important properties are "reindex", "refresh", "seed" etc. + * The order of properties is not relevant (but the order of children is). + * + * @param obj the input (is not changed by the method) + * @return a new JSON object + */ + public static JsonObject cleanedAndNormalized(JsonObject obj) { + obj = JsonObject.fromJson(obj.toString(), true); + obj.getProperties().remove(":version"); + obj.getProperties().remove(":nameSeed"); + obj.getProperties().remove(":mappingVersion"); + obj.getProperties().remove("refresh"); + obj.getProperties().remove("reindexCount"); + obj.getProperties().remove("reindex"); + obj.getProperties().remove("seed"); + obj.getProperties().remove("merges"); + obj.getProperties().remove("mergeInfo"); + obj.getProperties().remove("mergeChecksum"); + for (String p : new ArrayList<>(obj.getProperties().keySet())) { + if (p.endsWith("@lucene")) { + obj.getProperties().remove(p); + } else if (p.endsWith("@elasticsearch")) { + obj.getProperties().remove(p); + } else { + // remove "str:", "nam:", etc if needed + String v = obj.getProperties().get(p); + String v2 = normalizeOakString(v); + if (!v2.equals(v)) { + obj.getProperties().put(p, v2); + } + } + } + removeUUIDs(obj); + for (Entry e : obj.getChildren().entrySet()) { + obj.getChildren().put(e.getKey(), cleanedAndNormalized(e.getValue())); + } + // re-build the properties in alphabetical order + // (sorting the child nodes would be incorrect however, as order is significant here) + TreeMap props = new TreeMap<>(obj.getProperties()); + obj.getProperties().clear(); + for (Entry e : props.entrySet()) { + obj.getProperties().put(e.getKey(), e.getValue()); + } + return obj; + } + + /** + * "Normalize" a JSON string value. Remove any "nam:" and "dat:" and "str:" + * prefix in the value, because customers won't use them normally. (We want the + * diff to be as simple as possible). + * + * @param value the value (including double quotes; eg. "str:value") + * @return the normalized value (including double quotes) + */ + private static String normalizeOakString(String value) { + if (value == null || !value.startsWith("\"")) { + // ignore numbers + return value; + } + value = JsopTokenizer.decodeQuoted(value); + if (value.startsWith("str:") || value.startsWith("nam:") || value.startsWith("dat:")) { + value = value.substring("str:".length()); + } + return JsopBuilder.encode(value); + } + + /** + * Remove all "jcr:uuid" properties (including those in children), because the + * values might conflict. (new uuids are added later when needed). + * + * @param obj the JSON object where uuids will be removed. + */ + private static void removeUUIDs(JsonObject obj) { + obj.getProperties().remove("jcr:uuid"); + for (JsonObject c : obj.getChildren().values()) { + removeUUIDs(c); + } + } + + /** + * Merge a product index with a diff. If the product index is null, then the + * diff needs to contain a complete custom index definition. + * + * @param productIndex the product index definition, or null if none + * @param diff the diff (from the diff.index definition) + * @return the index definition of the merged index + */ + public JsonObject processMerge(JsonObject productIndex, JsonObject diff) { + JsonObject result; + if (productIndex == null) { + // fully custom index + result = new JsonObject(true); + } else { + result = JsonObject.fromJson(productIndex.toString(), true); + } + mergeInto("", diff, result); + addPrimaryType("", result); + return result; + } + + /** + * Add primary type properties where needed. For the top-level index definition, + * this is "oak:QueryIndexDefinition", and "nt:unstructured" elsewhere. + * + * @param path the path (so we can call the method recursively) + * @param json the JSON object (is changed if needed) + */ + private static void addPrimaryType(String path, JsonObject json) { + // all nodes need to have a node type; + // the index definition itself (at root level) is "oak:QueryIndexDefinition", + // and all other nodes are "nt:unstructured" + if (!json.getProperties().containsKey("jcr:primaryType")) { + // all nodes need to have a primary type, + // otherwise index import will fail + String nodeType; + if (path.isEmpty()) { + nodeType = "oak:QueryIndexDefinition"; + } else { + nodeType = "nt:unstructured"; + } + String nodeTypeValue = "nam:" + nodeType; + json.getProperties().put("jcr:primaryType", JsopBuilder.encode(nodeTypeValue)); + } + for (Entry e : json.getChildren().entrySet()) { + addPrimaryType(path + "/" + e.getKey(), e.getValue()); + } + } + + /** + * Merge a JSON diff into a target index definition. + * + * @param path the path + * @param diff the diff (what to merge) + * @param target where to merge into + */ + private void mergeInto(String path, JsonObject diff, JsonObject target) { + for (String p : diff.getProperties().keySet()) { + if (path.isEmpty()) { + if ("jcr:primaryType".equals(p)) { + continue; + } + } + if (target.getProperties().containsKey(p)) { + // we do not currently allow to overwrite most existing properties + if (p.equals("boost")) { + // allow overwriting the boost value + LOG.info("Overwrite property {} value at {}", p, path); + target.getProperties().put(p, diff.getProperties().get(p)); + } else { + LOG.warn("Ignoring existing property {} at {}", p, path); + } + } else { + target.getProperties().put(p, diff.getProperties().get(p)); + } + } + for (String c : diff.getChildren().keySet()) { + String targetChildName = c; + if (!target.getChildren().containsKey(c)) { + if (path.endsWith("/properties")) { + // search for a property with the same "name" value + String propertyName = diff.getChildren().get(c).getProperties().get("name"); + if (propertyName != null) { + propertyName = JsonNodeBuilder.oakStringValue(propertyName); + String c2 = getChildWithKeyValuePair(target, "name", propertyName); + if (c2 != null) { + targetChildName = c2; + } + } + // search for a property with the same "function" value + String function = diff.getChildren().get(c).getProperties().get("function"); + if (function != null) { + function = JsonNodeBuilder.oakStringValue(function); + String c2 = getChildWithKeyValuePair(target, "function", function); + if (c2 != null) { + targetChildName = c2; + } + } + } + if (targetChildName.equals(c)) { + // only create the child (properties are added below) + target.getChildren().put(c, new JsonObject()); + } + } + mergeInto(path + "/" + targetChildName, diff.getChildren().get(c), target.getChildren().get(targetChildName)); + } + if (target.getProperties().isEmpty() && target.getChildren().isEmpty()) { + if (deleteCreatesDummyIndex) { + // dummy index + target.getProperties().put("async", "\"async\""); + target.getProperties().put("includedPaths", "\"/dummy\""); + target.getProperties().put("queryPaths", "\"/dummy\""); + target.getProperties().put("type", "\"lucene\""); + JsopBuilder buff = new JsopBuilder(); + buff.object(). + key("properties").object(). + key("dummy").object(). + key("name").value("dummy"). + key("propertyIndex").value(true). + endObject(). + endObject(). + endObject(); + JsonObject indexRules = JsonObject.fromJson(buff.toString(), true); + target.getChildren().put("indexRules", indexRules); + } else { + target.getProperties().put("type", "\"disabled\""); + } + } + } + + public static String getChildWithKeyValuePair(JsonObject obj, String key, String value) { + for(Entry c : obj.getChildren().entrySet()) { + String v2 = c.getValue().getProperties().get(key); + if (v2 == null) { + continue; + } + v2 = JsonNodeBuilder.oakStringValue(v2); + if (value.equals(v2)) { + return c.getKey(); + } + } + return null; + } + + /** + * Compare two JSON object, ignoring the order of properties. (The order of + * children is however significant). + * + * This is done in addition to the checksum comparison, because the in theory + * the customer might change the checksum (it is not read-only as read-only + * values are not supported). We do not rely on the comparison, but if comparison + * and checksum comparison do not match, we log a warning. + * + * @param a the first object + * @param b the second object + * @return true if the keys and values are equal + */ + public boolean isSameIgnorePropertyOrder(JsonObject a, JsonObject b) { + if (!a.getChildren().keySet().equals(b.getChildren().keySet())) { + log("Child (order) difference: {} vs {}", + a.getChildren().keySet(), b.getChildren().keySet()); + return false; + } + for (String k : a.getChildren().keySet()) { + if (!isSameIgnorePropertyOrder( + a.getChildren().get(k), b.getChildren().get(k))) { + return false; + } + } + TreeMap pa = new TreeMap<>(a.getProperties()); + TreeMap pb = new TreeMap<>(b.getProperties()); + if (!pa.toString().equals(pb.toString())) { + log("Property value difference: {} vs {}", pa.toString(), pb.toString()); + } + return pa.toString().equals(pb.toString()); + } + + /** + * Read a diff.index from the repository, if it exists. + * This is needed because the build-transform job doesn't have this + * data: it is only available in the writeable repository. + * + * @param repositoryNodeStore the node store + * @return a map, possibly with a single entry with this key + */ + public Map readDiffIndex(NodeStore repositoryNodeStore, String name) { + HashMap map = new HashMap<>(); + NodeState root = repositoryNodeStore.getRoot(); + String indexPath = "/oak:index/" + name; + NodeState idxState = NodeStateUtils.getNode(root, indexPath); + log("Searching index {}: found={}", indexPath, idxState.exists()); + if (!idxState.exists()) { + return map; + } + JsopBuilder builder = new JsopBuilder(); + String filter = "{\"properties\":[\"*\", \"-:childOrder\"],\"nodes\":[\"*\", \"-:*\"]}"; + JsonSerializer serializer = new JsonSerializer(builder, filter, new Base64BlobSerializer()); + serializer.serialize(idxState); + JsonObject jsonObj = JsonObject.fromJson(builder.toString(), true); + jsonObj = cleanedAndNormalized(jsonObj); + log("Found {}", jsonObj.toString()); + map.put(indexPath, jsonObj); + return map; + } + + private void log(String format, Object... arguments) { + if (logAtInfoLevel) { + LOG.info(format, arguments); + } else { + LOG.debug(format, arguments); + } + } + +} diff --git a/oak-core/DiffIndexTest.java b/oak-core/DiffIndexTest.java new file mode 100644 index 00000000000..e5ae279a207 --- /dev/null +++ b/oak-core/DiffIndexTest.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; + +import org.apache.commons.io.IOUtils; +import org.apache.jackrabbit.JcrConstants; +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate; +import org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider; +import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.optimizer.DiffIndexUpdater; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.reference.ReferenceEditorProvider; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EditorHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.junit.Test; +import org.mockito.MockedStatic; + +/** + * Tests for DiffIndex functionality. + */ +public class DiffIndexTest { + + @Test + public void testFindMatchingIndexName() throws IOException { + String indexJson = "{\n" + + " \"index\": {\n" + + " \"compatVersion\": 2,\n" + + " \"async\": \"async\",\n" + + " \"queryPaths\": [\"/content/dam/test\"],\n" + + " \"includedPaths\": [\"/content/dam/test\"],\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"type\": \"lucene\",\n" + + " \"tags\": [\"fragments\"],\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"title\": {\n" + + " \"name\": \"str:jcr:title\",\n" + + " \"propertyIndex\": true,\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + try (MockedStatic mockedStatic = mockStatic(RootIndexesListService.class)) { + NodeStore store = mock(NodeStore.class); + + String indexesJsonString; + + try (InputStream stream = getClass().getResourceAsStream("/org/apache/jackrabbit/oak/plugins/index/diff/indexes.json")) { + indexesJsonString = IOUtils.toString(stream, StandardCharsets.UTF_8); + } + + mockedStatic.when(() -> RootIndexesListService.getRootIndexDefinitions(eq(store), anyString())) + .thenReturn(JsonObject.fromJson(indexesJsonString, true)); + + Optional matchingIndexName = DiffIndexUpdater.findMatchingIndexName(store, indexJson); + + assertTrue(matchingIndexName.isPresent()); + } + } + + @Test + public void listIndexes() { + NodeStore store = new MemoryNodeStore(INITIAL_CONTENT); + JsonObject indexDefs = RootIndexesListService.getRootIndexDefinitions(store, "property"); + // expect at least one index + assertFalse(indexDefs.getChildren().isEmpty()); + } + + @Test + public void tryReadStringNull() { + assertNull(DiffIndex.tryReadString(null)); + } + + @Test + public void tryReadStringValidContent() { + String content = "Hello, World!"; + PropertyState prop = BinaryPropertyState.binaryProperty("jcr:data", + content.getBytes(StandardCharsets.UTF_8)); + assertEquals(content, DiffIndex.tryReadString(prop)); + } + + @Test + public void tryReadStringEmpty() { + PropertyState prop = BinaryPropertyState.binaryProperty("jcr:data", new byte[0]); + assertEquals("", DiffIndex.tryReadString(prop)); + } + + @Test + public void tryReadStringJsonContent() { + String content = "{ \"key\": \"value\", \"array\": [1, 2, 3] }"; + PropertyState prop = BinaryPropertyState.binaryProperty("jcr:data", + content.getBytes(StandardCharsets.UTF_8)); + assertEquals(content, DiffIndex.tryReadString(prop)); + } + + @Test + public void tryReadStringIOException() throws IOException { + PropertyState prop = mock(PropertyState.class); + Blob blob = mock(Blob.class); + InputStream failingStream = new InputStream() { + @Override + public int read() throws IOException { + throw new IOException("Simulated read failure"); + } + @Override + public byte[] readAllBytes() throws IOException { + throw new IOException("Simulated read failure"); + } + }; + when(prop.getValue(Type.BINARY)).thenReturn(blob); + when(blob.getNewStream()).thenReturn(failingStream); + + // Should return null (not throw exception) + assertNull(DiffIndex.tryReadString(prop)); + } + + @Test + public void testDiffIndexUpdate() throws Exception { + // Create a memory node store + NodeStore store = new MemoryNodeStore(INITIAL_CONTENT); + + storeDiff(store, "2026-01-01T00:00:00.000Z", "" + + "{ \"acme.testIndex\": {\n" + + " \"async\": [ \"async\", \"nrt\" ],\n" + + " \"compatVersion\": 2,\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"includedPaths\": [ \"/content/dam\" ],\n" + + " \"jcr:primaryType\": \"oak:QueryIndexDefinition\",\n" + + " \"queryPaths\": [ \"/content/dam\" ],\n" + + " \"selectionPolicy\": \"tag\",\n" + + " \"tags\": [ \"abc\" ],\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"created\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"name\": \"str:jcr:created\",\n" + + " \"ordered\": true,\n" + + " \"propertyIndex\": true,\n" + + " \"type\": \"Date\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " } }"); + + JsonObject repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(store, "lucene"); + assertSameJson("{\n" + + " \"/oak:index/acme.testIndex-1-custom-1\": {\n" + + " \"compatVersion\": 2,\n" + + " \"async\": [\"async\", \"nrt\"],\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"mergeChecksum\": \"34e7f7f0eb480ea781317b56134bc85fc59ed97031d95f518fdcff230aec28a2\",\n" + + " \"mergeInfo\": \"This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html\",\n" + + " \"selectionPolicy\": \"tag\",\n" + + " \"queryPaths\": [\"/content/dam\"],\n" + + " \"includedPaths\": [\"/content/dam\"],\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"tags\": [\"abc\"],\n" + + " \"merges\": [\"/oak:index/acme.testIndex\"],\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"created\": {\n" + + " \"ordered\": true,\n" + + " \"name\": \"str:jcr:created\",\n" + + " \"propertyIndex\": true,\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"type\": \"Date\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", repositoryDefinitions.toString()); + + storeDiff(store, "2026-01-01T00:00:00.001Z", "" + + "{ \"acme.testIndex\": {\n" + + " \"async\": [ \"async\", \"nrt\" ],\n" + + " \"compatVersion\": 2,\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"includedPaths\": [ \"/content/dam\" ],\n" + + " \"jcr:primaryType\": \"oak:QueryIndexDefinition\",\n" + + " \"queryPaths\": [ \"/content/dam\" ],\n" + + " \"selectionPolicy\": \"tag\",\n" + + " \"tags\": [ \"abc\" ],\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"created\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"name\": \"str:jcr:created\",\n" + + " \"propertyIndex\": true\n" + + " },\n" + + " \"modified\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"name\": \"str:jcr:modified\",\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " } }"); + + repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(store, "lucene"); + assertSameJson("{\n" + + " \"/oak:index/acme.testIndex-1-custom-2\": {\n" + + " \"compatVersion\": 2,\n" + + " \"async\": [\"async\", \"nrt\"],\n" + + " \"mergeChecksum\": \"41df9c87e4d4fca446aed3f55e6d188304a2cb49bae442b75403dc23a89b266f\",\n" + + " \"mergeInfo\": \"This index was auto-merged. See also https://oak-indexing.github.io/oakTools/simplified.html\",\n" + + " \"selectionPolicy\": \"tag\",\n" + + " \"queryPaths\": [\"/content/dam\"],\n" + + " \"includedPaths\": [\"/content/dam\"],\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"type\": \"lucene\",\n" + + " \"tags\": [\"abc\"],\n" + + " \"merges\": [\"/oak:index/acme.testIndex\"],\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"created\": {\n" + + " \"name\": \"str:jcr:created\",\n" + + " \"propertyIndex\": true,\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\"\n" + + " },\n" + + " \"modified\": {\n" + + " \"name\": \"str:jcr:modified\",\n" + + " \"propertyIndex\": true,\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", repositoryDefinitions.toString()); + + storeDiff(store, "2026-01-01T00:00:00.002Z", "" + + "{}"); + + repositoryDefinitions = RootIndexesListService.getRootIndexDefinitions(store, "lucene"); + assertSameJson("{}", repositoryDefinitions.toString()); + } + + private void assertSameJson(String a, String b) { + JsonObject ja = JsonObject.fromJson(a, true); + JsonObject jb = JsonObject.fromJson(b, true); + if (!DiffIndexMerger.instance().isSameIgnorePropertyOrder(ja, jb)) { + assertEquals(a, b); + } + } + + private void storeDiff(NodeStore store, String timestamp, String json) throws CommitFailedException { + // Get the root builder + NodeBuilder builder = store.getRoot().builder(); + + List indexEditors = List.of( + new ReferenceEditorProvider(), new PropertyIndexEditorProvider(), new NodeCounterEditorProvider()); + IndexEditorProvider provider = CompositeIndexEditorProvider.compose(indexEditors); + EditorHook hook = new EditorHook(new IndexUpdateProvider(provider)); + + // Create the index definition at /oak:index/diff.index + NodeBuilder indexDefs = builder.child(INDEX_DEFINITIONS_NAME); + NodeBuilder diffIndex = indexDefs.child("diff.index"); + + // Set index properties + diffIndex.setProperty("jcr:primaryType", IndexConstants.INDEX_DEFINITIONS_NODE_TYPE, Type.NAME); + diffIndex.setProperty(TYPE_PROPERTY_NAME, "disabled"); + + // Create the diff.json child node with primary type nt:file + NodeBuilder diffJson = diffIndex.child("diff.json"); + diffJson.setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_FILE, Type.NAME); + + // Create jcr:content child node (required for nt:file) with empty text + NodeBuilder content = diffJson.child(JcrConstants.JCR_CONTENT); + content.setProperty(JcrConstants.JCR_LASTMODIFIED, timestamp); + content.setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_RESOURCE, Type.NAME); + + content.setProperty("jcr:data", json); + + // Merge changes to the store + store.merge(builder, hook, CommitInfo.EMPTY); + + // Run async indexing explicitly + for (int i = 0; i < 5; i++) { + try (AsyncIndexUpdate async = new AsyncIndexUpdate("async", store, provider)) { + async.run(); + } + } + } +} + diff --git a/oak-core/MergeTest.java b/oak-core/MergeTest.java new file mode 100644 index 00000000000..c55d1c3ab6c --- /dev/null +++ b/oak-core/MergeTest.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.diff; + +import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.commons.json.JsonObject; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.junit.Test; + +public class MergeTest { + + // test that we can extract the file from the diff.json node (just that) + @Test + public void extractFile() { + JsonObject indexDiff = JsonObject.fromJson("{\n" + + " \"damAssetLucene\": {\n" + + " \"indexRules\": {\n" + + " \"dam:Asset\": {\n" + + " \"properties\": {\n" + + " \"y\": {\n" + + " \"name\": \"y\",\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true); + String indexDiffString = indexDiff.toString(); + String base64Prop = + "\":blobId:" + Base64.getEncoder().encodeToString(indexDiffString.getBytes(StandardCharsets.UTF_8)) + "\""; + JsonObject repositoryDefinitions = JsonObject.fromJson("{\n" + + " \"/oak:index/damAssetLucene-12\": {\n" + + " \"jcr:primaryType\": \"oak:IndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"async\": [\"async\", \"nrt\"],\n" + + " \"tags\": [\"abc\"],\n" + + " \"includedPaths\": \"/content/dam\",\n" + + " \"indexRules\": {\n" + + " \"dam:Asset\": {\n" + + " \"properties\": {\n" + + " \"x\": {\n" + + " \"name\": \"x\",\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"/oak:index/diff.index\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"type\": \"lucene\", \"includedPaths\": \"/same\", \"queryPaths\": \"/same\",\n" + + " \"diff.json\": {\n" + + " \"jcr:primaryType\": \"nam:nt:file\",\n" + + " \"jcr:content\": {\n" + + " \"jcr:primaryType\": \"nam:nt:resource\",\n" + + " \"jcr:mimeType\": \"application/json\",\n" + + " \"jcr:data\":\n" + + " " + base64Prop + "\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true); + + HashMap target = new HashMap<>(); + DiffIndexMerger.tryExtractDiffIndex(repositoryDefinitions, "/oak:index/diff.index", target); + assertEquals("{damAssetLucene={\n" + + " \"indexRules\": {\n" + + " \"dam:Asset\": {\n" + + " \"properties\": {\n" + + " \"y\": {\n" + + " \"name\": \"y\",\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}}", target.toString()); + } + + @Test + public void renamedProperty() { + // A property might be indexed twice, by adding two children to the "properties" node + // that both have the same "name" value. + // Alternatively, they could have the same "function" value. + String merged = DiffIndexMerger.instance().processMerge(JsonObject.fromJson("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"name\": \"test\",\n" + + " \"boost\": 1.0\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }" + + "", true), JsonObject.fromJson("{\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"def\": {\n" + + " \"name\": \"test\",\n" + + " \"boost\": 1.2\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true)).toString(); + assertEquals("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"name\": \"test\",\n" + + " \"boost\": 1.2\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", merged); + } + + @Test + public void renamedFunction() { + // A function might be indexed twice, by adding two children to the "properties" node + // that both have the same "function" value. + String merged = DiffIndexMerger.instance().processMerge(JsonObject.fromJson("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"function\": \"upper(test)\",\n" + + " \"boost\": 1.0\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }" + + "", true), JsonObject.fromJson("{\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"def\": {\n" + + " \"function\": \"upper(test)\",\n" + + " \"boost\": 1.2\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true)).toString(); + assertEquals("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"function\": \"upper(test)\",\n" + + " \"boost\": 1.2\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", merged); + } + + @Test + public void createDummy() { + // when enabling "deleteCreatesDummyIndex", then a dummy index is created + // (that indexes /dummy, which doesn't exist) + String merged = new DiffIndexMerger(new String[0], true, true, false).processMerge(JsonObject.fromJson("{}" + + "", true), JsonObject.fromJson("{}", true)).toString(); + assertEquals("{\n" + + " \"async\": \"async\",\n" + + " \"includedPaths\": \"/dummy\",\n" + + " \"queryPaths\": \"/dummy\",\n" + + " \"type\": \"lucene\",\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"dummy\": {\n" + + " \"name\": \"dummy\",\n" + + " \"propertyIndex\": true,\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}", merged); + } + + @Test + public void boost() { + // - "analyzed" must not be overwritten + // - "ordered" is added + // - "boost" is overwritten + String merged = DiffIndexMerger.instance().processMerge(JsonObject.fromJson("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"analyzed\": true,\n" + + " \"boost\": 1.0\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }" + + "", true), JsonObject.fromJson("{\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"abc\": {\n" + + " \"analyzed\": false,\n" + + " \"ordered\": true,\n" + + " \"boost\": 1.2\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true)).toString(); + assertEquals("{\n" + + " \"jcr:primaryType\": \"nam:oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"acme:Test\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"abc\": {\n" + + " \"jcr:primaryType\": \"nam:nt:unstructured\",\n" + + " \"analyzed\": true,\n" + + " \"boost\": 1.2,\n" + + " \"ordered\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", merged); + } + + @Test + public void mergeDiffsTest() { + JsonObject a = JsonObject.fromJson("{\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"prop1\": {\n" + + " \"name\": \"field1\",\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"type\": \"lucene\"\n" + + " }", true); + JsonObject b = JsonObject.fromJson("{\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"prop2\": {\n" + + " \"name\": \"field2\",\n" + + " \"ordered\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " },\n" + + " \"async\": [\"async\", \"nrt\"]\n" + + " }", true); + String merged = DiffIndexMerger.mergeDiffs(a, b).toString(); + assertEquals("{\n" + + " \"type\": \"lucene\",\n" + + " \"async\": [\"async\", \"nrt\"],\n" + + " \"indexRules\": {\n" + + " \"acme:Test\": {\n" + + " \"properties\": {\n" + + " \"prop1\": {\n" + + " \"name\": \"field1\",\n" + + " \"propertyIndex\": true\n" + + " },\n" + + " \"prop2\": {\n" + + " \"name\": \"field2\",\n" + + " \"ordered\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", merged); + } + + @Test + public void switchToLuceneChildrenTest() { + JsonObject indexDef = JsonObject.fromJson("{\n" + + " \"type\": \"elasticsearch\",\n" + + " \"type@lucene\": \"lucene\",\n" + + " \"async@lucene\": \"[\\\"async\\\", \\\"nrt\\\"]\",\n" + + " \"async\": \"[\\\"async\\\"]\",\n" + + " \"codec@lucene\": \"Lucene46\",\n" + + " \"indexRules\": {\n" + + " \"dam:Asset\": {\n" + + " \"properties\": {\n" + + " \"test\": {\n" + + " \"name\": \"jcr:content/metadata/test\",\n" + + " \"boost@lucene\": \"2.0\",\n" + + " \"boost\": \"1.0\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + " }", true); + DiffIndexMerger.switchToLuceneChildren(indexDef); + String result = indexDef.toString(); + assertEquals("{\n" + + " \"type\": \"lucene\",\n" + + " \"async\": \"[\\\"async\\\", \\\"nrt\\\"]\",\n" + + " \"codec\": \"Lucene46\",\n" + + " \"indexRules\": {\n" + + " \"dam:Asset\": {\n" + + " \"properties\": {\n" + + " \"test\": {\n" + + " \"name\": \"jcr:content/metadata/test\",\n" + + " \"boost\": \"2.0\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", result); + } + + @Test + public void includesUnsupportedPathsTest() { + DiffIndexMerger merger = new DiffIndexMerger(new String[]{"/apps", "/libs"}, false, false, false); + + assertEquals(true, merger.includesUnsupportedPaths(null)); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/apps"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/apps/acme"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/apps/acme/test"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/libs"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/libs/foundation"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/content", "/apps"})); + assertEquals(true, merger.includesUnsupportedPaths(new String[]{"/content", "/libs/test"})); + + assertEquals(false, merger.includesUnsupportedPaths(new String[]{"/content"})); + assertEquals(false, merger.includesUnsupportedPaths(new String[]{"/content/dam"})); + assertEquals(false, merger.includesUnsupportedPaths(new String[]{"/var"})); + assertEquals(false, merger.includesUnsupportedPaths(new String[]{"/etc"})); + assertEquals(false, merger.includesUnsupportedPaths(new String[]{"/content", "/var", "/etc"})); + } + + @Test + public void readDiffIndexTest() throws CommitFailedException { + NodeStore store = new MemoryNodeStore(INITIAL_CONTENT); + NodeBuilder root = store.getRoot().builder(); + NodeBuilder oakIndex = root.child("oak:index"); + NodeBuilder diffIndex = oakIndex.child("diff.index.optimizer"); + diffIndex.setProperty("jcr:primaryType", "nt:unstructured"); + diffIndex.setProperty("type", "lucene"); + diffIndex.setProperty("async", "async"); + diffIndex.setProperty("includedPaths", "/content"); + NodeBuilder indexRules = diffIndex.child("indexRules"); + NodeBuilder damAsset = indexRules.child("dam:Asset"); + NodeBuilder properties = damAsset.child("properties"); + NodeBuilder testProp = properties.child("test"); + testProp.setProperty("name", "jcr:content/metadata/test"); + testProp.setProperty("propertyIndex", true); + store.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY); + + Map result = DiffIndexMerger.instance().readDiffIndex(store, "diff.index.optimizer"); + + assertEquals(1, result.size()); + assertTrue(result.containsKey("/oak:index/diff.index.optimizer")); + JsonObject indexDef = result.get("/oak:index/diff.index.optimizer"); + assertEquals("\"lucene\"", indexDef.getProperties().get("type")); + assertEquals("\"async\"", indexDef.getProperties().get("async")); + assertEquals("\"/content\"", indexDef.getProperties().get("includedPaths")); + assertTrue(indexDef.getChildren().containsKey("indexRules")); + } +} diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java index 161558320a6..aab502b8fb1 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -139,7 +139,7 @@ private static void processDiffs(NodeStore store, NodeBuilder indexDefinitions, } JsonObject newDef = diffs.getChildren().get(indexPath); String indexName = PathUtils.getName(indexPath); - JsonNodeBuilder.addOrReplace(indexDefinitions, store, indexName, + JsonNodeUpdater.addOrReplace(indexDefinitions, store, indexName, IndexConstants.INDEX_DEFINITIONS_NODE_TYPE, newDef.toString()); updateNodetypeIndexForPath(indexDefinitions, indexName, true); disableOrRemoveOldVersions(indexDefinitions, indexPath, indexName); diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java index 07c0d872e9d..7183828db92 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndexMerger.java @@ -201,7 +201,7 @@ public static String tryExtractDiffIndex(JsonObject indexDefs, String name, Hash LOG.warn(message); return message; } - String jcrData = JsonNodeBuilder.oakStringValue(jcrContent, "jcr:data"); + String jcrData = JsonNodeUpdater.oakStringValue(jcrContent, "jcr:data"); try { diff = JsonObject.fromJson(jcrData, true); } catch (Exception e) { @@ -342,7 +342,7 @@ public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject n // there is no customization (any more), which means a dummy index may be needed log("No customization for {}", indexName); } else { - includedPaths = JsonNodeBuilder.oakStringArrayValue(indexDiff, "includedPaths"); + includedPaths = JsonNodeUpdater.oakStringArrayValue(indexDiff, "includedPaths"); if (includesUnsupportedPaths(includedPaths)) { LOG.warn("New custom index {} is not supported because it contains an unsupported path ({})", indexName, Arrays.toString(unsupportedIncludedPaths)); @@ -350,7 +350,7 @@ public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject n } } } else { - includedPaths = JsonNodeBuilder.oakStringArrayValue(latestProductIndex, "includedPaths"); + includedPaths = JsonNodeUpdater.oakStringArrayValue(latestProductIndex, "includedPaths"); if (includesUnsupportedPaths(includedPaths)) { LOG.warn("Customizing index {} is not supported because it contains an unsupported path ({})", latestProductKey, Arrays.toString(unsupportedIncludedPaths)); @@ -387,7 +387,7 @@ public boolean processMerge(String indexName, JsonObject indexDiff, JsonObject n // new index key = prefix + indexName + "-1-custom-1"; } else { - String latestMergeChecksum = JsonNodeBuilder.oakStringValue(latestIndexVersion, "mergeChecksum"); + String latestMergeChecksum = JsonNodeUpdater.oakStringValue(latestIndexVersion, "mergeChecksum"); JsonObject latestDef = cleanedAndNormalized(switchToLucene(latestIndexVersion)); if (isSameIgnorePropertyOrder(mergedDef, latestDef)) { // normal case: no change @@ -512,7 +512,7 @@ private static String computeMergeChecksum(JsonObject json) { */ public static JsonObject switchToLucene(JsonObject indexDef) { JsonObject obj = JsonObject.fromJson(indexDef.toString(), true); - String type = JsonNodeBuilder.oakStringValue(obj, "type"); + String type = JsonNodeUpdater.oakStringValue(obj, "type"); if (type == null || !"elasticsearch".equals(type) ) { return obj; } @@ -700,7 +700,7 @@ private void mergeInto(String path, JsonObject diff, JsonObject target) { // search for a property with the same "name" value String propertyName = diff.getChildren().get(c).getProperties().get("name"); if (propertyName != null) { - propertyName = JsonNodeBuilder.oakStringValue(propertyName); + propertyName = JsonNodeUpdater.oakStringValue(propertyName); String c2 = getChildWithKeyValuePair(target, "name", propertyName); if (c2 != null) { targetChildName = c2; @@ -709,7 +709,7 @@ private void mergeInto(String path, JsonObject diff, JsonObject target) { // search for a property with the same "function" value String function = diff.getChildren().get(c).getProperties().get("function"); if (function != null) { - function = JsonNodeBuilder.oakStringValue(function); + function = JsonNodeUpdater.oakStringValue(function); String c2 = getChildWithKeyValuePair(target, "function", function); if (c2 != null) { targetChildName = c2; @@ -753,7 +753,7 @@ public static String getChildWithKeyValuePair(JsonObject obj, String key, String if (v2 == null) { continue; } - v2 = JsonNodeBuilder.oakStringValue(v2); + v2 = JsonNodeUpdater.oakStringValue(v2); if (value.equals(v2)) { return c.getKey(); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeUpdater.java similarity index 95% rename from oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java rename to oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeUpdater.java index da34188b7c4..a5dabdbce7f 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilder.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeUpdater.java @@ -25,6 +25,7 @@ import java.util.Base64; import java.util.UUID; +import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; @@ -58,9 +59,9 @@ * * "null" entries are not supported. */ -public class JsonNodeBuilder { +public class JsonNodeUpdater { - private static final Logger LOG = LoggerFactory.getLogger(JsonNodeBuilder.class); + private static final Logger LOG = LoggerFactory.getLogger(JsonNodeUpdater.class); /** * Add a replace a node, including all child nodes, in the node store. @@ -80,8 +81,8 @@ public static void addOrReplace(NodeBuilder builder, NodeStore nodeStore, String JsonObject json = JsonObject.fromJson(jsonString, true); for (String name : PathUtils.elements(targetPath)) { NodeBuilder child = builder.child(name); - if (!child.hasProperty("jcr:primaryType")) { - child.setProperty("jcr:primaryType", nodeType, Type.NAME); + if (!child.hasProperty(JCR_PRIMARYTYPE)) { + child.setProperty(JCR_PRIMARYTYPE, nodeType, Type.NAME); } builder = child; } @@ -106,11 +107,11 @@ private static void storeConfigNode(NodeStore nodeStore, NodeBuilder builder, St String v = e.getValue(); storeConfigProperty(nodeStore, builder, k, v); } - if (!json.getProperties().containsKey("jcr:primaryType")) { - builder.setProperty("jcr:primaryType", nodeType, Type.NAME); + if (!json.getProperties().containsKey(JCR_PRIMARYTYPE)) { + builder.setProperty(JCR_PRIMARYTYPE, nodeType, Type.NAME); } for (PropertyState prop : builder.getProperties()) { - if ("jcr:primaryType".equals(prop.getName())) { + if (JCR_PRIMARYTYPE.equals(prop.getName())) { continue; } if (!json.getProperties().containsKey(prop.getName())) { @@ -118,7 +119,7 @@ private static void storeConfigNode(NodeStore nodeStore, NodeBuilder builder, St } } builder.setProperty(TreeConstants.OAK_CHILD_ORDER, childOrder, Type.NAMES); - if ("nt:resource".equals(JsonNodeBuilder.oakStringValue(json, "jcr:primaryType"))) { + if ("nt:resource".equals(JsonNodeUpdater.oakStringValue(json, JCR_PRIMARYTYPE))) { if (!json.getProperties().containsKey("jcr:uuid")) { String uuid = UUID.randomUUID().toString(); builder.setProperty("jcr:uuid", uuid); @@ -145,7 +146,7 @@ private static void storeConfigProperty(NodeStore nodeStore, NodeBuilder builder if (value.startsWith("str:") || value.startsWith("nam:") || value.startsWith("dat:")) { value = value.substring("str:".length()); } - if ("jcr:primaryType".equals(propertyName)) { + if (JCR_PRIMARYTYPE.equals(propertyName)) { builder.setProperty(propertyName, value, Type.NAME); } else { builder.setProperty(propertyName, value); diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java index 9a80fa8a7e1..ea29bb23a3e 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java @@ -57,7 +57,7 @@ public void addNodeTypeAndUUID() throws CommitFailedException, IOException { + " }\n" + " }", true); NodeBuilder builder = ns.getRoot().builder(); - JsonNodeBuilder.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); + JsonNodeUpdater.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); ns.merge(builder, new EmptyHook(), CommitInfo.EMPTY); String json2 = JsonUtils.nodeStateToJson(ns.getRoot(), 5); json2 = json2.replaceAll("jcr:uuid\" : \".*\"", "jcr:uuid\" : \"...\""); @@ -87,7 +87,7 @@ public void addNodeTypeAndUUID() throws CommitFailedException, IOException { "\"double2\":1.0," + "\"child2\":{\"y\":2}}", true); builder = ns.getRoot().builder(); - JsonNodeBuilder.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); + JsonNodeUpdater.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); ns.merge(builder, new EmptyHook(), CommitInfo.EMPTY); assertEquals("{\n" + " \"test\" : {\n" @@ -115,7 +115,7 @@ public void store() throws CommitFailedException, IOException { "\"child\":{\"x\":1}," + "\"blob\":\":blobId:dGVzdA==\"}", true); NodeBuilder builder = ns.getRoot().builder(); - JsonNodeBuilder.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); + JsonNodeUpdater.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); ns.merge(builder, new EmptyHook(), CommitInfo.EMPTY); assertEquals("{\n" + " \"test\" : {\n" @@ -139,7 +139,7 @@ public void store() throws CommitFailedException, IOException { "\"double2\":1.0," + "\"child2\":{\"y\":2}}", true); builder = ns.getRoot().builder(); - JsonNodeBuilder.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); + JsonNodeUpdater.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); ns.merge(builder, new EmptyHook(), CommitInfo.EMPTY); assertEquals("{\n" + " \"test\" : {\n" @@ -158,42 +158,42 @@ public void store() throws CommitFailedException, IOException { @Test public void oakStringValue() { - assertEquals("123", JsonNodeBuilder.oakStringValue("123")); - assertEquals("45.67", JsonNodeBuilder.oakStringValue("45.67")); - assertEquals("-10", JsonNodeBuilder.oakStringValue("-10")); + assertEquals("123", JsonNodeUpdater.oakStringValue("123")); + assertEquals("45.67", JsonNodeUpdater.oakStringValue("45.67")); + assertEquals("-10", JsonNodeUpdater.oakStringValue("-10")); String helloBase64 = Base64.getEncoder().encodeToString("hello".getBytes(StandardCharsets.UTF_8)); - assertEquals("hello", JsonNodeBuilder.oakStringValue("\":blobId:" + helloBase64 + "\"")); + assertEquals("hello", JsonNodeUpdater.oakStringValue("\":blobId:" + helloBase64 + "\"")); - assertEquals("hello", JsonNodeBuilder.oakStringValue("\"str:hello\"")); - assertEquals("acme:Test", JsonNodeBuilder.oakStringValue("\"nam:acme:Test\"")); - assertEquals("2024-01-19", JsonNodeBuilder.oakStringValue("\"dat:2024-01-19\"")); + assertEquals("hello", JsonNodeUpdater.oakStringValue("\"str:hello\"")); + assertEquals("acme:Test", JsonNodeUpdater.oakStringValue("\"nam:acme:Test\"")); + assertEquals("2024-01-19", JsonNodeUpdater.oakStringValue("\"dat:2024-01-19\"")); } @Test public void getStringSet() { - assertNull(JsonNodeBuilder.getStringSet(null)); - assertEquals(new TreeSet<>(Arrays.asList("hello")), JsonNodeBuilder.getStringSet("\"hello\"")); - assertEquals(null, JsonNodeBuilder.getStringSet("123")); - assertEquals(new TreeSet<>(Arrays.asList("content/abc")), JsonNodeBuilder.getStringSet("\"content\\/abc\"")); - assertTrue(JsonNodeBuilder.getStringSet("[]").isEmpty()); - assertEquals(new TreeSet<>(Arrays.asList("a")), JsonNodeBuilder.getStringSet("[\"a\"]")); - assertEquals(new TreeSet<>(Arrays.asList("content/abc")), JsonNodeBuilder.getStringSet("[\"content\\/abc\"]")); - assertEquals(new TreeSet<>(Arrays.asList("a")), JsonNodeBuilder.getStringSet("[\"a\",\"a\"]")); - assertEquals(new TreeSet<>(Arrays.asList("a", "z")), JsonNodeBuilder.getStringSet("[\"z\",\"a\"]")); + assertNull(JsonNodeUpdater.getStringSet(null)); + assertEquals(new TreeSet<>(Arrays.asList("hello")), JsonNodeUpdater.getStringSet("\"hello\"")); + assertEquals(null, JsonNodeUpdater.getStringSet("123")); + assertEquals(new TreeSet<>(Arrays.asList("content/abc")), JsonNodeUpdater.getStringSet("\"content\\/abc\"")); + assertTrue(JsonNodeUpdater.getStringSet("[]").isEmpty()); + assertEquals(new TreeSet<>(Arrays.asList("a")), JsonNodeUpdater.getStringSet("[\"a\"]")); + assertEquals(new TreeSet<>(Arrays.asList("content/abc")), JsonNodeUpdater.getStringSet("[\"content\\/abc\"]")); + assertEquals(new TreeSet<>(Arrays.asList("a")), JsonNodeUpdater.getStringSet("[\"a\",\"a\"]")); + assertEquals(new TreeSet<>(Arrays.asList("a", "z")), JsonNodeUpdater.getStringSet("[\"z\",\"a\"]")); } @Test public void oakStringArrayValue() throws IOException { - assertNull(JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{}", true), "p")); - assertArrayEquals(new String[]{"hello"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":\"hello\"}", true), "p")); - assertNull(JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":123}", true), "p")); - assertArrayEquals(new String[]{"content/abc"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":\"content\\/abc\"}", true), "p")); - assertArrayEquals(new String[]{}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":[]}", true), "p")); - assertArrayEquals(new String[]{"a"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"a\"]}", true), "p")); - assertArrayEquals(new String[]{"content/abc"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"content\\/abc\"]}", true), "p")); - assertArrayEquals(new String[]{"a"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"a\",\"a\"]}", true), "p")); - assertArrayEquals(new String[]{"a", "z"}, JsonNodeBuilder.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"z\",\"a\"]}", true), "p")); + assertNull(JsonNodeUpdater.oakStringArrayValue(JsonObject.fromJson("{}", true), "p")); + assertArrayEquals(new String[]{"hello"}, JsonNodeUpdater.oakStringArrayValue(JsonObject.fromJson("{\"p\":\"hello\"}", true), "p")); + assertNull(JsonNodeUpdater.oakStringArrayValue(JsonObject.fromJson("{\"p\":123}", true), "p")); + assertArrayEquals(new String[]{"content/abc"}, JsonNodeUpdater.oakStringArrayValue(JsonObject.fromJson("{\"p\":\"content\\/abc\"}", true), "p")); + assertArrayEquals(new String[]{}, JsonNodeUpdater.oakStringArrayValue(JsonObject.fromJson("{\"p\":[]}", true), "p")); + assertArrayEquals(new String[]{"a"}, JsonNodeUpdater.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"a\"]}", true), "p")); + assertArrayEquals(new String[]{"content/abc"}, JsonNodeUpdater.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"content\\/abc\"]}", true), "p")); + assertArrayEquals(new String[]{"a"}, JsonNodeUpdater.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"a\",\"a\"]}", true), "p")); + assertArrayEquals(new String[]{"a", "z"}, JsonNodeUpdater.oakStringArrayValue(JsonObject.fromJson("{\"p\":[\"z\",\"a\"]}", true), "p")); } @Test @@ -207,7 +207,7 @@ public void addOrReplacePrefixesBooleansAndEscapes() throws CommitFailedExceptio "\"boolFalse\":false," + "\"escapedArray\":[\"\\/content\\/path\"]}", true); NodeBuilder builder = ns.getRoot().builder(); - JsonNodeBuilder.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); + JsonNodeUpdater.addOrReplace(builder, ns, "/test", "nt:test", json.toString()); ns.merge(builder, new EmptyHook(), CommitInfo.EMPTY); assertEquals("{\n" + " \"test\" : {\n" From fc19b5a4aa7d91eeae4a2e6d69c5ff35d12a7660 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Mon, 9 Feb 2026 17:14:56 +0100 Subject: [PATCH 16/16] OAK-12010 Simplified index management --- .../apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java | 3 +++ .../{JsonNodeBuilderTest.java => JsonNodeUpdaterTest.java} | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) rename oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/{JsonNodeBuilderTest.java => JsonNodeUpdaterTest.java} (99%) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java index aab502b8fb1..6df72e2ce83 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diff/DiffIndex.java @@ -116,6 +116,9 @@ public static JsonObject collectDiffs(NodeBuilder indexDefinitions) { LOG.warn("{}: {}", message, e.getMessage(), e); diffIndexDefinition.setProperty("error", message + ": " + e.getMessage()); } + if (!diffIndexDefinition.hasProperty("info")) { + diffIndexDefinition.setProperty("info", "This diff is are automatically merged with other indexes. See https://oak-indexing.github.io/oakTools/simplified.html"); + } } return diffs; } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeUpdaterTest.java similarity index 99% rename from oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java rename to oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeUpdaterTest.java index ea29bb23a3e..4bcc5db181e 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeBuilderTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/diff/JsonNodeUpdaterTest.java @@ -36,7 +36,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.junit.Test; -public class JsonNodeBuilderTest { +public class JsonNodeUpdaterTest { @Test public void addNodeTypeAndUUID() throws CommitFailedException, IOException {