Skip to content

Upgrade Lucene module to version 10.3.2 #19096

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ashu17188 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Upgrade Lucene module to version 10.3.2 #19096

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions lucene/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,13 @@
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<artifactId>lucene-analysis-common</artifactId>
<version>${lucene.version}</version>
</dependency>
</dependencies>

<properties>
<lucene.version>7.4.0</lucene.version>
<lucene.version>10.3.2</lucene.version>
</properties>

</project>
10 changes: 7 additions & 3 deletions lucene/src/main/java/com/baeldung/lucene/InMemoryLuceneIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
Expand Down Expand Up @@ -65,9 +66,10 @@ public List<Document> searchIndex(String inField, String queryString) {
IndexReader indexReader = DirectoryReader.open(memoryIndex);
IndexSearcher searcher = new IndexSearcher(indexReader);
TopDocs topDocs = searcher.search(query, 10);
StoredFields storedFields = searcher.storedFields();
List<Document> documents = new ArrayList<>();
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
documents.add(searcher.doc(scoreDoc.doc));
documents.add(storedFields.document(scoreDoc.doc));
}

return documents;
Expand All @@ -94,9 +96,10 @@ public List<Document> searchIndex(Query query) {
IndexReader indexReader = DirectoryReader.open(memoryIndex);
IndexSearcher searcher = new IndexSearcher(indexReader);
TopDocs topDocs = searcher.search(query, 10);
StoredFields storedFields = searcher.storedFields();
List<Document> documents = new ArrayList<>();
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
documents.add(searcher.doc(scoreDoc.doc));
documents.add(storedFields.document(scoreDoc.doc));
}

return documents;
Expand All @@ -112,9 +115,10 @@ public List<Document> searchIndex(Query query, Sort sort) {
IndexReader indexReader = DirectoryReader.open(memoryIndex);
IndexSearcher searcher = new IndexSearcher(indexReader);
TopDocs topDocs = searcher.search(query, 10, sort);
StoredFields storedFields = searcher.storedFields();
List<Document> documents = new ArrayList<>();
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
documents.add(searcher.doc(scoreDoc.doc));
documents.add(storedFields.document(scoreDoc.doc));
}

return documents;
Expand Down
4 changes: 3 additions & 1 deletion lucene/src/main/java/com/baeldung/lucene/LuceneFileSearch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
Expand Down Expand Up @@ -62,9 +63,10 @@ public List<Document> searchFiles(String inField, String queryString) {
IndexReader indexReader = DirectoryReader.open(indexDirectory);
IndexSearcher searcher = new IndexSearcher(indexReader);
TopDocs topDocs = searcher.search(query, 10);
StoredFields storedFields = searcher.storedFields();
List<Document> documents = new ArrayList<>();
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
documents.add(searcher.doc(scoreDoc.doc));
documents.add(storedFields.document(scoreDoc.doc));
}

return documents;
Expand Down
11 changes: 5 additions & 6 deletions lucene/src/main/java/com/baeldung/lucene/MyCustomAnalyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,19 @@
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.miscellaneous.CapitalizationFilter;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;

public class MyCustomAnalyzer extends Analyzer{
public class MyCustomAnalyzer extends Analyzer {

@Override
protected TokenStreamComponents createComponents(String fieldName) {
final StandardTokenizer src = new StandardTokenizer();
TokenStream result = new StandardFilter(src);
result = new LowerCaseFilter(result);
result = new StopFilter(result, StandardAnalyzer.STOP_WORDS_SET);
// StandardFilter was removed in Lucene 10 - no longer needed
TokenStream result = new LowerCaseFilter(src);
result = new StopFilter(result, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET);
result = new PorterStemFilter(result);
result = new CapitalizationFilter(result);
return new TokenStreamComponents(src, result);
Expand Down
11 changes: 6 additions & 5 deletions lucene/src/test/java/com/baeldung/lucene/LuceneAnalyzerIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.junit.Test;

public class LuceneAnalyzerIntegrationTest {
Expand All @@ -37,12 +37,13 @@ public class LuceneAnalyzerIntegrationTest {
public void whenUseStandardAnalyzer_thenAnalyzed() throws IOException {
List<String> result = analyze(SAMPLE_TEXT, new StandardAnalyzer());

assertThat(result, contains("baeldung.com", "lucene", "analyzers", "test"));
// In Lucene 10, StandardAnalyzer no longer filters stop words by default
assertThat(result, contains("this", "is", "baeldung.com", "lucene", "analyzers", "test"));
}

@Test
public void whenUseStopAnalyzer_thenAnalyzed() throws IOException {
List<String> result = analyze(SAMPLE_TEXT, new StopAnalyzer());
List<String> result = analyze(SAMPLE_TEXT, new StopAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET));

assertThat(result, contains("baeldung", "com", "lucene", "analyzers", "test"));
}
Expand Down Expand Up @@ -100,7 +101,7 @@ public void whenUseCustomAnalyzer_thenAnalyzed() throws IOException {

@Test
public void givenTermQuery_whenUseCustomAnalyzer_thenCorrect() {
InMemoryLuceneIndex luceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new MyCustomAnalyzer());
InMemoryLuceneIndex luceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), new MyCustomAnalyzer());
luceneIndex.indexDocument("introduction", "introduction to lucene");
luceneIndex.indexDocument("analyzers", "guide to lucene analyzers");
Query query = new TermQuery(new Term("body", "Introduct"));
Expand All @@ -117,7 +118,7 @@ public void givenTermQuery_whenUsePerFieldAnalyzerWrapper_thenCorrect() {

PerFieldAnalyzerWrapper wrapper =
new PerFieldAnalyzerWrapper(new StandardAnalyzer(), analyzerMap);
InMemoryLuceneIndex luceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), wrapper);
InMemoryLuceneIndex luceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), wrapper);
luceneIndex.indexDocument("introduction", "introduction to lucene");
luceneIndex.indexDocument("analyzers", "guide to lucene analyzers");

Expand Down
22 changes: 11 additions & 11 deletions lucene/src/test/java/com/baeldung/lucene/LuceneInMemorySearchIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.util.BytesRef;
import org.junit.Assert;
import org.junit.Test;
Expand All @@ -24,7 +24,7 @@ public class LuceneInMemorySearchIntegrationTest {

@Test
public void givenSearchQueryWhenFetchedDocumentThenCorrect() {
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), new StandardAnalyzer());
inMemoryLuceneIndex.indexDocument("Hello world", "Some hello world ");

List<Document> documents = inMemoryLuceneIndex.searchIndex("body", "world");
Expand All @@ -34,7 +34,7 @@ public void givenSearchQueryWhenFetchedDocumentThenCorrect() {

@Test
public void givenTermQueryWhenFetchedDocumentThenCorrect() {
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), new StandardAnalyzer());
inMemoryLuceneIndex.indexDocument("activity", "running in track");
inMemoryLuceneIndex.indexDocument("activity", "Cars are running on road");

Expand All @@ -47,7 +47,7 @@ public void givenTermQueryWhenFetchedDocumentThenCorrect() {

@Test
public void givenPrefixQueryWhenFetchedDocumentThenCorrect() {
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), new StandardAnalyzer());
inMemoryLuceneIndex.indexDocument("article", "Lucene introduction");
inMemoryLuceneIndex.indexDocument("article", "Introduction to Lucene");

Expand All @@ -60,7 +60,7 @@ public void givenPrefixQueryWhenFetchedDocumentThenCorrect() {

@Test
public void givenBooleanQueryWhenFetchedDocumentThenCorrect() {
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), new StandardAnalyzer());
inMemoryLuceneIndex.indexDocument("Destination", "Las Vegas singapore car");
inMemoryLuceneIndex.indexDocument("Commutes in singapore", "Bus Car Bikes");

Expand All @@ -79,7 +79,7 @@ public void givenBooleanQueryWhenFetchedDocumentThenCorrect() {

@Test
public void givenPhraseQueryWhenFetchedDocumentThenCorrect() {
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), new StandardAnalyzer());
inMemoryLuceneIndex.indexDocument("quotes", "A rose by any other name would smell as sweet.");

Query query = new PhraseQuery(1, "body", new BytesRef("smell"), new BytesRef("sweet"));
Expand All @@ -90,7 +90,7 @@ public void givenPhraseQueryWhenFetchedDocumentThenCorrect() {

@Test
public void givenFuzzyQueryWhenFetchedDocumentThenCorrect() {
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), new StandardAnalyzer());
inMemoryLuceneIndex.indexDocument("article", "Halloween Festival");
inMemoryLuceneIndex.indexDocument("decoration", "Decorations for Halloween");

Expand All @@ -103,7 +103,7 @@ public void givenFuzzyQueryWhenFetchedDocumentThenCorrect() {

@Test
public void givenWildCardQueryWhenFetchedDocumentThenCorrect() {
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), new StandardAnalyzer());
inMemoryLuceneIndex.indexDocument("article", "Lucene introduction");
inMemoryLuceneIndex.indexDocument("article", "Introducing Lucene with Spring");

Expand All @@ -116,7 +116,7 @@ public void givenWildCardQueryWhenFetchedDocumentThenCorrect() {

@Test
public void givenSortFieldWhenSortedThenCorrect() {
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), new StandardAnalyzer());
inMemoryLuceneIndex.indexDocument("Ganges", "River in India");
inMemoryLuceneIndex.indexDocument("Mekong", "This river flows in south Asia");
inMemoryLuceneIndex.indexDocument("Amazon", "Rain forest river");
Expand All @@ -126,7 +126,7 @@ public void givenSortFieldWhenSortedThenCorrect() {
Term term = new Term("body", "river");
Query query = new WildcardQuery(term);

SortField sortField = new SortField("title", SortField.Type.STRING_VAL, false);
SortField sortField = new SortField("title", SortField.Type.STRING, false);
Sort sortByTitle = new Sort(sortField);

List<Document> documents = inMemoryLuceneIndex.searchIndex(query, sortByTitle);
Expand All @@ -136,7 +136,7 @@ public void givenSortFieldWhenSortedThenCorrect() {

@Test
public void whenDocumentDeletedThenCorrect() {
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new RAMDirectory(), new StandardAnalyzer());
InMemoryLuceneIndex inMemoryLuceneIndex = new InMemoryLuceneIndex(new ByteBuffersDirectory(), new StandardAnalyzer());
inMemoryLuceneIndex.indexDocument("Ganges", "River in India");
inMemoryLuceneIndex.indexDocument("Mekong", "This river flows in south Asia");

Expand Down