nouveau: Upgrade to Lucene 10

nouveau/README.md

@@ -3,8 +3,8 @@
 Nouveau is a modern replacement for dreyfus/clouseau and is built on;
 
 1) the Dropwizard framework (https://dropwizard.io)
-2) Java 11+
-3) Lucene 9
+2) Java 21+
+3) Lucene 10
 
 Nouveau transforms Apache CouchDB databases into Apache Lucene indexes at the shard level and then merges the results together.
 
@@ -23,7 +23,7 @@ This work is currently EXPERIMENTAL and may change in ways that invalidate any e
 * integration with resharding
 * update=false
 * `_nouveau_info`
-* `_search_cleanup`
+* `_nouveau_cleanup`
 * /openapi.{json.yaml}
 
 ## What doesn't work yet?

nouveau/build.gradle

@@ -23,7 +23,7 @@ dependencies {
     implementation 'io.dropwizard.metrics:metrics-jersey2'
     testImplementation 'io.dropwizard:dropwizard-testing'
 
-    def luceneVersion = '9.12.1'
+    def luceneVersion = '10.3.0'
     implementation group: 'org.apache.lucene', name: 'lucene-core', version: luceneVersion
     implementation group: 'org.apache.lucene', name: 'lucene-queryparser', version: luceneVersion
     implementation group: 'org.apache.lucene', name: 'lucene-analysis-common', version: luceneVersion
@@ -46,7 +46,7 @@ group = 'org.apache.couchdb'
 version = '1.0-SNAPSHOT'
 
 java {
-    sourceCompatibility = "11"
+    sourceCompatibility = "21"
 }
 
 jar {

nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java

@@ -24,8 +24,8 @@
 import org.apache.couchdb.nouveau.core.UserAgentFilter;
 import org.apache.couchdb.nouveau.health.AnalyzeHealthCheck;
 import org.apache.couchdb.nouveau.health.IndexHealthCheck;
-import org.apache.couchdb.nouveau.lucene9.Lucene9Module;
-import org.apache.couchdb.nouveau.lucene9.ParallelSearcherFactory;
+import org.apache.couchdb.nouveau.lucene.LuceneModule;
+import org.apache.couchdb.nouveau.lucene.ParallelSearcherFactory;
 import org.apache.couchdb.nouveau.resources.AnalyzeResource;
 import org.apache.couchdb.nouveau.resources.IndexResource;
 import org.apache.couchdb.nouveau.tasks.CloseAllIndexesTask;
@@ -65,7 +65,7 @@ public void run(NouveauApplicationConfiguration configuration, Environment envir
         environment.lifecycle().manage(indexManager);
 
         // Serialization classes
-        environment.getObjectMapper().registerModule(new Lucene9Module());
+        environment.getObjectMapper().registerModule(new LuceneModule());
 
         // AnalyzeResource
         final AnalyzeResource analyzeResource = new AnalyzeResource();

nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java

@@ -16,12 +16,21 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.PropertyNamingStrategies;
 import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import jakarta.validation.constraints.Max;
+import jakarta.validation.constraints.Min;
 import jakarta.validation.constraints.NotEmpty;
 import java.util.Map;
 
 @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class)
 public class IndexDefinition {
 
+    public static final int LATEST_LUCENE_VERSION = 10;
+    public static final int LEGACY_LUCENE_VERSION = 9;
+
+    @Max(LATEST_LUCENE_VERSION)
+    @Min(LEGACY_LUCENE_VERSION)
+    private int luceneVersion = LEGACY_LUCENE_VERSION; // Legacy version if not set.
+
     @NotEmpty
     private String defaultAnalyzer;
 
@@ -31,11 +40,22 @@ public IndexDefinition() {
         // Jackson deserialization
     }
 
-    public IndexDefinition(final String defaultAnalyzer, final Map<String, String> fieldAnalyzers) {
+    public IndexDefinition(
+            final int luceneVersion, final String defaultAnalyzer, final Map<String, String> fieldAnalyzers) {
+        this.luceneVersion = luceneVersion;
         this.defaultAnalyzer = defaultAnalyzer;
         this.fieldAnalyzers = fieldAnalyzers;
     }
 
+    @JsonProperty
+    public int getLuceneVersion() {
+        return luceneVersion;
+    }
+
+    public void setLuceneVersion(int luceneVersion) {
+        this.luceneVersion = luceneVersion;
+    }
+
     @JsonProperty
     public String getDefaultAnalyzer() {
         return defaultAnalyzer;
@@ -62,6 +82,7 @@ public boolean hasFieldAnalyzers() {
     public int hashCode() {
         final int prime = 31;
         int result = 1;
+        result = prime * result + luceneVersion;
         result = prime * result + ((defaultAnalyzer == null) ? 0 : defaultAnalyzer.hashCode());
         result = prime * result + ((fieldAnalyzers == null) ? 0 : fieldAnalyzers.hashCode());
         return result;
@@ -73,6 +94,7 @@ public boolean equals(Object obj) {
         if (obj == null) return false;
         if (getClass() != obj.getClass()) return false;
         IndexDefinition other = (IndexDefinition) obj;
+        if (luceneVersion != other.luceneVersion) return false;
         if (defaultAnalyzer == null) {
             if (other.defaultAnalyzer != null) return false;
         } else if (!defaultAnalyzer.equals(other.defaultAnalyzer)) return false;
@@ -84,6 +106,7 @@ public boolean equals(Object obj) {
 
     @Override
     public String toString() {
-        return "IndexDefinition [defaultAnalyzer=" + defaultAnalyzer + ", fieldAnalyzers=" + fieldAnalyzers + "]";
+        return "IndexDefinition [luceneVersion=" + luceneVersion + ", defaultAnalyzer=" + defaultAnalyzer
+                + ", fieldAnalyzers=" + fieldAnalyzers + "]";
     }
 }

nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java

@@ -34,16 +34,19 @@
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.couchdb.nouveau.api.IndexDefinition;
-import org.apache.couchdb.nouveau.lucene9.Lucene9AnalyzerFactory;
-import org.apache.couchdb.nouveau.lucene9.Lucene9Index;
+import org.apache.couchdb.nouveau.lucene.LuceneAnalyzerFactory;
+import org.apache.couchdb.nouveau.lucene.LuceneIndex;
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.backward_codecs.lucene912.Lucene912Codec;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.misc.store.DirectIODirectory;
 import org.apache.lucene.search.SearcherFactory;
 import org.apache.lucene.search.SearcherManager;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -218,7 +221,6 @@ public void create(final String name, IndexDefinition indexDefinition) throws IO
             assertSame(indexDefinition, loadIndexDefinition(name));
             return;
         }
-
         final Lock lock = this.createLock.writeLock(name);
         lock.lock();
         try {
@@ -392,15 +394,22 @@ private Index load(final String name) throws IOException {
         LOGGER.info("opening {}", name);
         final Path path = indexPath(name);
         final IndexDefinition indexDefinition = loadIndexDefinition(name);
-        final Analyzer analyzer = Lucene9AnalyzerFactory.fromDefinition(indexDefinition);
-        final Directory dir = new DirectIODirectory(FSDirectory.open(path.resolve("9")));
+        final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition);
+        final int luceneVersion = indexDefinition.getLuceneVersion();
+        final Directory dir = new DirectIODirectory(FSDirectory.open(path.resolve(Integer.toString(luceneVersion))));
         final IndexWriterConfig config = new IndexWriterConfig(analyzer);
+        if (luceneVersion != Version.LATEST.major) {
+            config.setOpenMode(OpenMode.APPEND);
+        }
+        if (luceneVersion == IndexDefinition.LEGACY_LUCENE_VERSION) {
+            config.setCodec(new Lucene912Codec());
+        }
         config.setUseCompoundFile(false);
         final IndexWriter writer = new IndexWriter(dir, config);
         final long updateSeq = getSeq(writer, "update_seq");
         final long purgeSeq = getSeq(writer, "purge_seq");
         final SearcherManager searcherManager = new SearcherManager(writer, searcherFactory);
-        return new Lucene9Index(analyzer, writer, updateSeq, purgeSeq, searcherManager);
+        return new LuceneIndex(analyzer, writer, updateSeq, purgeSeq, searcherManager);
     }
 
     private long getSeq(final IndexWriter writer, final String key) throws IOException {

nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java

@@ -21,6 +21,7 @@
 import org.apache.couchdb.nouveau.api.SearchRequest;
 import org.apache.couchdb.nouveau.api.SearchResults;
 import org.apache.couchdb.nouveau.resources.IndexResource;
+import org.apache.lucene.util.Version;
 
 public final class IndexHealthCheck extends HealthCheck {
 
@@ -32,14 +33,14 @@ public IndexHealthCheck(final IndexResource indexResource) {
 
     @Override
     protected Result check() throws Exception {
-        final String name = "___test9";
+        final String name = "___test" + Version.LATEST.major;
         try {
             indexResource.deletePath(name, null);
         } catch (IOException e) {
             // Ignored, index might not exist yet.
         }
 
-        indexResource.createIndex(name, new IndexDefinition("standard", null));
+        indexResource.createIndex(name, new IndexDefinition(Version.LATEST.major, "standard", null));
         try {
             final DocumentUpdateRequest documentUpdateRequest =
                     new DocumentUpdateRequest(0, 1, null, Collections.emptyList());

nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactory.java → nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactory.java

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package org.apache.couchdb.nouveau.lucene9;
+package org.apache.couchdb.nouveau.lucene;
 
 import jakarta.ws.rs.WebApplicationException;
 import jakarta.ws.rs.core.Response.Status;
@@ -60,9 +60,9 @@
 import org.apache.lucene.analysis.th.ThaiAnalyzer;
 import org.apache.lucene.analysis.tr.TurkishAnalyzer;
 
-public final class Lucene9AnalyzerFactory {
+public final class LuceneAnalyzerFactory {
 
-    private Lucene9AnalyzerFactory() {}
+    private LuceneAnalyzerFactory() {}
 
     public static Analyzer fromDefinition(final IndexDefinition indexDefinition) {
         final Analyzer defaultAnalyzer = newAnalyzer(indexDefinition.getDefaultAnalyzer());

nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java → nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package org.apache.couchdb.nouveau.lucene9;
+package org.apache.couchdb.nouveau.lucene;
 
 import jakarta.ws.rs.WebApplicationException;
 import jakarta.ws.rs.core.Response.Status;
@@ -97,7 +97,7 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 
-public class Lucene9Index extends Index {
+public class LuceneIndex extends Index {
 
     private static final Sort DEFAULT_SORT =
             new Sort(SortField.FIELD_SCORE, new SortField("_id", SortField.Type.STRING));
@@ -106,9 +106,9 @@ public class Lucene9Index extends Index {
     private final Analyzer analyzer;
     private final IndexWriter writer;
     private final SearcherManager searcherManager;
-    private final Lucene9IndexSchema schema;
+    private final LuceneIndexSchema schema;
 
-    public Lucene9Index(
+    public LuceneIndex(
             final Analyzer analyzer,
             final IndexWriter writer,
             final long updateSeq,
@@ -290,8 +290,8 @@ private void collectHits(final IndexSearcher searcher, final TopDocs topDocs, fi
             hits.add(new SearchHit(doc.get("_id"), after, fields));
         }
 
-        searchResults.setTotalHits(topDocs.totalHits.value);
-        searchResults.setTotalHitsRelation(topDocs.totalHits.relation);
+        searchResults.setTotalHits(topDocs.totalHits.value());
+        searchResults.setTotalHitsRelation(topDocs.totalHits.relation());
         searchResults.setHits(hits);
     }
 
@@ -537,22 +537,22 @@ private Query parse(final SearchRequest request) {
         return result;
     }
 
-    private Lucene9IndexSchema initSchema(IndexWriter writer) {
+    private LuceneIndexSchema initSchema(IndexWriter writer) {
         var commitData = writer.getLiveCommitData();
         if (commitData == null) {
-            return Lucene9IndexSchema.emptySchema();
+            return LuceneIndexSchema.emptySchema();
         }
         for (var entry : commitData) {
             if (entry.getKey().equals("_schema")) {
-                return Lucene9IndexSchema.fromString(entry.getValue());
+                return LuceneIndexSchema.fromString(entry.getValue());
             }
         }
-        return Lucene9IndexSchema.emptySchema();
+        return LuceneIndexSchema.emptySchema();
     }
 
     @Override
     public String toString() {
-        return "Lucene9Index [analyzer=" + analyzer + ", writer=" + writer + ", searcherManager=" + searcherManager
+        return "LuceneIndex [analyzer=" + analyzer + ", writer=" + writer + ", searcherManager=" + searcherManager
                 + "]";
     }
 

nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexSchema.java → nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndexSchema.java

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package org.apache.couchdb.nouveau.lucene9;
+package org.apache.couchdb.nouveau.lucene;
 
 import jakarta.ws.rs.WebApplicationException;
 import jakarta.ws.rs.core.Response.Status;
@@ -32,7 +32,7 @@
 import org.apache.couchdb.nouveau.api.TextField;
 import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig;
 
-final class Lucene9IndexSchema {
+final class LuceneIndexSchema {
 
     public enum Type {
         STRING,
@@ -56,23 +56,23 @@ private static Type fromField(final Field field) {
 
     private final ConcurrentMap<String, Type> map;
 
-    private Lucene9IndexSchema(Map<String, Type> map) {
+    private LuceneIndexSchema(Map<String, Type> map) {
         this.map = new ConcurrentHashMap<>(map);
         this.map.put("_id", Type.STRING);
     }
 
-    public static Lucene9IndexSchema emptySchema() {
-        return new Lucene9IndexSchema(new HashMap<String, Type>());
+    public static LuceneIndexSchema emptySchema() {
+        return new LuceneIndexSchema(new HashMap<String, Type>());
     }
 
-    public static Lucene9IndexSchema fromString(final String schemaStr) {
+    public static LuceneIndexSchema fromString(final String schemaStr) {
         Objects.requireNonNull(schemaStr);
         if (schemaStr.isEmpty()) {
             return emptySchema();
         }
         var map = Arrays.stream(schemaStr.split(","))
                 .collect(Collectors.toMap(i -> i.split(":")[0], i -> Type.valueOf(i.split(":")[1])));
-        return new Lucene9IndexSchema(map);
+        return new LuceneIndexSchema(map);
     }
 
     public void update(final Collection<Field> fields) {

nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Module.java → nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneModule.java

@@ -11,16 +11,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package org.apache.couchdb.nouveau.lucene9;
+package org.apache.couchdb.nouveau.lucene;
 
 import com.fasterxml.jackson.core.Version;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import org.apache.lucene.search.Query;
 
-public class Lucene9Module extends SimpleModule {
+public class LuceneModule extends SimpleModule {
 
-    public Lucene9Module() {
-        super("lucene9", Version.unknownVersion());
+    public LuceneModule() {
+        super("lucene", Version.unknownVersion());
 
         // Query
         addSerializer(Query.class, new QuerySerializer());

nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java → nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParser.java

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package org.apache.couchdb.nouveau.lucene9;
+package org.apache.couchdb.nouveau.lucene;
 
 import java.util.Map;
 import org.apache.lucene.analysis.Analyzer;

nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/ParallelSearcherFactory.java → nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/ParallelSearcherFactory.java

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package org.apache.couchdb.nouveau.lucene9;
+package org.apache.couchdb.nouveau.lucene;
 
 import java.io.IOException;
 import java.util.concurrent.Executor;

nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QueryDeserializer.java → nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QueryDeserializer.java

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package org.apache.couchdb.nouveau.lucene9;
+package org.apache.couchdb.nouveau.lucene;
 
 import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.JsonParser;