Skip to content

Commit 48826fd

Browse files
elahrvivazelahrvivaz
authored
GEOMESA-3556 Fix SimpleFeatureTypes geometry index flag parsing (#3513)
1 parent 8bdb7c6 commit 48826fd

File tree

6 files changed

+118
-48
lines changed

6 files changed

+118
-48
lines changed

‎docs/user/datastores/index_creation.rst‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,11 @@ Examples
9696
import org.locationtech.geomesa.utils.interop.SimpleFeatureTypes;
9797

9898
// creates a default attribute index on name and an implicit default z3 and z2 index on geom
99-
String spec = "name:String:index=true,dtg:Date,*geom:Point:srid=4325";
99+
String spec = "name:String:index=true,dtg:Date,*geom:Point:srid=4326";
100100
// creates an attribute index on name (with a secondary date index), and a z3 index on geom and dtg
101-
spec = "name:String:index='attr:dtg',dtg:Date,*geom:Point:srid=4325:index='z3:dtg'";
101+
spec = "name:String:index='attr:dtg',dtg:Date,*geom:Point:srid=4326:index='z3:dtg'";
102102
// creates an attribute index on name (with a secondary date index), and a z3 index on geom and dtg and disables the ID index
103-
spec = "name:String:index='attr:dtg',dtg:Date,*geom:Point:srid=4325:index='z3:dtg';id.index.enabled=false";
103+
spec = "name:String:index='attr:dtg',dtg:Date,*geom:Point:srid=4326:index='z3:dtg';id.index.enabled=false";
104104

105105
SimpleFeatureType sft = SimpleFeatureTypes.createType("myType", spec);
106106
// alternatively, set user data after parsing the type string (but before calling "createSchema")

‎geomesa-accumulo/geomesa-accumulo-datastore/src/test/scala/org/locationtech/geomesa/accumulo/index/ConfigurableIndexesTest.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ class ConfigurableIndexesTest extends Specification with TestWithFeatureType {
7979

8080
"add another empty index" >> {
8181
import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType
82-
val updated = SimpleFeatureTypes.mutable(sft)
82+
val updated = SimpleFeatureTypes.copy(sft)
8383
updated.setIndices(sft.getIndices :+ IndexId(Z2Index.name, Z2Index.version, Seq("geom"), IndexMode.ReadWrite))
8484
ds.updateSchema(sftName, updated)
8585
val indices = ds.manager.indices(updated)

‎geomesa-index-api/src/main/scala/org/locationtech/geomesa/index/geotools/GeoMesaDataStore.scala‎

Lines changed: 20 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ package org.locationtech.geomesa.index.geotools
1111
import com.github.benmanes.caffeine.cache.{AsyncCacheLoader, AsyncLoadingCache, CacheLoader, Caffeine}
1212
import com.typesafe.scalalogging.LazyLogging
1313
import org.geotools.api.data._
14-
import org.geotools.api.feature.`type`.AttributeDescriptor
1514
import org.geotools.api.feature.simple.SimpleFeatureType
1615
import org.geotools.api.filter.Filter
1716
import org.locationtech.geomesa.index.FlushableFeatureWriter
@@ -20,7 +19,6 @@ import org.locationtech.geomesa.index.conf.QueryHints
2019
import org.locationtech.geomesa.index.conf.partition.TablePartition
2120
import org.locationtech.geomesa.index.geotools.GeoMesaDataStore.{SchemaCompatibility, VersionKey}
2221
import org.locationtech.geomesa.index.geotools.GeoMesaDataStoreFactory.GeoMesaDataStoreConfig
23-
import org.locationtech.geomesa.index.index.attribute.AttributeIndex
2422
import org.locationtech.geomesa.index.index.id.IdIndex
2523
import org.locationtech.geomesa.index.planning.DataStoreQueryRunner
2624
import org.locationtech.geomesa.index.stats.HasGeoMesaStats
@@ -37,7 +35,7 @@ import org.locationtech.geomesa.utils.index.IndexMode
3735
import org.locationtech.geomesa.utils.io.CloseWithLogging
3836

3937
import java.io.IOException
40-
import java.util.Collections
38+
import java.util.{Collections, Locale}
4139
import java.util.concurrent.{ConcurrentHashMap, TimeUnit}
4240
import scala.util.Try
4341
import scala.util.control.NonFatal
@@ -503,42 +501,29 @@ abstract class GeoMesaDataStore[DS <: GeoMesaDataStore[DS]](val config: GeoMesaD
503501

504502
def remapCol(name: String): String = colMap.getOrElse(name, name)
505503

506-
// check for attributes flagged 'index' and convert them to sft-level user data
507-
def indexed(d: AttributeDescriptor): Boolean = {
508-
d.getUserData.get(AttributeOptions.OptIndex) match {
509-
case i: String if Seq("true", "full").exists(_.equalsIgnoreCase(i)) => true
510-
case i if i == null || Seq("false", "none").exists(_.equalsIgnoreCase(i.toString)) => false
511-
case i => throw new IllegalArgumentException(s"Configured index coverage '$i' is not valid: expected 'true'")
512-
}
513-
}
514-
sft.getAttributeDescriptors.asScala.foreach { d =>
515-
if (indexed(d)) {
516-
val existing = {
517-
val explicit = sft.getIndices
518-
if (explicit.nonEmpty) { explicit } else { previous.getIndices }
504+
val indices = {
505+
// set directly in the sft
506+
val explicitIndices = (previous.getIndices ++ sft.getIndices).map(i => i.copy(attributes = i.attributes.map(remapCol)))
507+
// set in index flags or geomesa.indices.enabled
508+
val configuredIndices =
509+
// don't add in a fid index unless explicitly enabled
510+
if (sft.getUserData.get(Configs.EnableFidIndex) == null &&
511+
Option(sft.getUserData.get(Configs.EnabledIndices))
512+
.forall(!_.toString.split(",").map(_.trim.toLowerCase(Locale.US)).exists(_.matches(s"^${IdIndex.name}:?")))) {
513+
GeoMesaFeatureIndexFactory.indices(sft).filterNot(_.name == IdIndex.name)
514+
} else {
515+
GeoMesaFeatureIndexFactory.indices(sft)
519516
}
520-
if (!existing.exists(e => e.name == AttributeIndex.name && remapCol(e.attributes.head) == d.getLocalName)) {
521-
val fields = Seq(d.getLocalName) ++ Option(sft.getGeomField) ++ sft.getDtgField
522-
val id = IndexId(AttributeIndex.name, AttributeIndex.version, fields, IndexMode.ReadWrite)
523-
sft.setIndices(existing :+ id)
517+
// get the unique indices - note: ignores index version
518+
(explicitIndices ++ configuredIndices).foldLeft(Seq.empty[IndexId]) { (sum, next) =>
519+
if (sum.exists(i => i.name == next.name && i.attributes == next.attributes)) {
520+
sum
521+
} else {
522+
sum :+ next
524523
}
525524
}
526525
}
527-
528-
// check for new indices and 'enabled indices' changes
529-
val indices = {
530-
val enabled = if (!sft.getUserData.containsKey(Configs.EnabledIndices)) { Seq.empty } else {
531-
GeoMesaFeatureIndexFactory.indices(sft)
532-
}
533-
val remapped = (previous.getIndices ++ sft.getIndices).map(i => i.copy(attributes = i.attributes.map(remapCol)))
534-
(remapped ++ enabled).foldLeft(Seq.empty[IndexId]) { (sum, next) =>
535-
// note: ignore index version
536-
if (sum.exists(i => i.name == next.name && i.attributes == next.attributes)) { sum } else { sum :+ next }
537-
}
538-
}
539-
if (indices != previous.getIndices) {
540-
sft.setIndices(indices)
541-
}
526+
sft.setIndices(indices)
542527

543528
// preserve any existing user data but overwrite any keys we redefine
544529
val userData = new java.util.HashMap[AnyRef, AnyRef](previous.getUserData)

‎geomesa-index-api/src/test/scala/org/locationtech/geomesa/index/index/ConfigurableIndicesTest.scala‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/***********************************************************************
2+
* Copyright (c) 2013-2025 General Atomics Integrated Intelligence, Inc.
3+
* All rights reserved. This program and the accompanying materials
4+
* are made available under the terms of the Apache License, Version 2.0
5+
* which accompanies this distribution and is available at
6+
* https://www.apache.org/licenses/LICENSE-2.0
7+
***********************************************************************/
8+
9+
package org.locationtech.geomesa.index.index
10+
11+
import com.typesafe.config.ConfigFactory
12+
import com.typesafe.scalalogging.LazyLogging
13+
import org.geotools.api.feature.simple.SimpleFeatureType
14+
import org.junit.runner.RunWith
15+
import org.locationtech.geomesa.index.TestGeoMesaDataStore
16+
import org.locationtech.geomesa.utils.geotools.{SchemaBuilder, SimpleFeatureTypes}
17+
import org.locationtech.geomesa.utils.io.WithClose
18+
import org.specs2.mutable.Specification
19+
import org.specs2.runner.JUnitRunner
20+
21+
@RunWith(classOf[JUnitRunner])
22+
class ConfigurableIndicesTest extends Specification with LazyLogging {
23+
24+
import org.locationtech.geomesa.utils.geotools.RichSimpleFeatureType.RichSimpleFeatureType
25+
26+
def getIndexConfig(sft: SimpleFeatureType): Seq[String] =
27+
sft.getIndices.map(i => s"${i.name}=${i.attributes.mkString(":")}").sorted
28+
29+
"GeoMesaDataStore" should {
30+
31+
// tests here are the examples from the docs
32+
33+
"support configurable indices in a spec string" in {
34+
WithClose(new TestGeoMesaDataStore(true)) { ds =>
35+
// creates a default attribute index on name and an implicit default z3 and z2 index on geom
36+
ds.createSchema(SimpleFeatureTypes.createType("test1", "name:String:index=true,dtg:Date,*geom:Point:srid=4326"))
37+
// creates an attribute index on name (with a secondary date index), and a z3 index on geom and dtg
38+
ds.createSchema(SimpleFeatureTypes.createType("test2", "name:String:index='attr:dtg',dtg:Date,*geom:Point:srid=4326:index='z3:dtg'"))
39+
// creates an attribute index on name (with a secondary date index), and a z3 index on geom and dtg and disables the ID index
40+
ds.createSchema(SimpleFeatureTypes.createType("test3", "name:String:index='attr:dtg',dtg:Date,*geom:Point:srid=4326:index='z3:dtg';id.index.enabled=false"))
41+
getIndexConfig(ds.getSchema("test1")) mustEqual Seq("attr=name:geom:dtg", "id=", "z2=geom", "z3=geom:dtg")
42+
getIndexConfig(ds.getSchema("test2")) mustEqual Seq("attr=name:dtg", "id=", "z3=geom:dtg")
43+
getIndexConfig(ds.getSchema("test3")) mustEqual Seq("attr=name:dtg", "z3=geom:dtg")
44+
}
45+
}
46+
47+
"support configurable indices with SchemaBuilder" in {
48+
val sft =
49+
SchemaBuilder.builder()
50+
.addString("name").withIndex("attr:dtg") // creates an attribute index on name, with a secondary date index
51+
.addInt("age").withIndex() // creates an attribute index on age, with a default secondary index
52+
.addDate("dtg") // not a primary index
53+
.addPoint("geom", default = true).withIndices("z3:dtg", "z2") // creates a z3 index with dtg, and a z2 index
54+
.userData
55+
.disableIdIndex() // disables the ID index
56+
.build("test1")
57+
WithClose(new TestGeoMesaDataStore(true)) { ds =>
58+
ds.createSchema(sft)
59+
getIndexConfig(ds.getSchema("test1")) mustEqual Seq("attr=age:geom:dtg", "attr=name:dtg", "z2=geom", "z3=geom:dtg")
60+
}
61+
}
62+
63+
"support configurable indices with config" in {
64+
val config =
65+
s"""{
66+
| type-name = test1
67+
| attributes = [
68+
| { name = "name", type = "String", index = "attr:dtg" } // creates an attribute index on name, with a secondary date index
69+
| { name = "age", type = "Int", index = "true" } // creates an attribute index on age, with a default secondary index
70+
| { name = "dtg", type = "Date" } // not a primary index
71+
| { name = "geom", type = "Point", srid = "4326", index = "z3:dtg,z2" } // creates a z3 index with dtg, and a z2 index
72+
| ]
73+
| user-data = {
74+
| "id.index.enabled" = "false" // disables the default ID index
75+
| }
76+
|}
77+
|""".stripMargin
78+
val sft = SimpleFeatureTypes.createType(ConfigFactory.parseString(config), path = None)
79+
WithClose(new TestGeoMesaDataStore(true)) { ds =>
80+
ds.createSchema(sft)
81+
getIndexConfig(ds.getSchema("test1")) mustEqual Seq("attr=age:geom:dtg", "attr=name:dtg", "z2=geom", "z3=geom:dtg")
82+
}
83+
}
84+
}
85+
}

‎geomesa-utils-parent/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/geotools/sft/SimpleFeatureSpec.scala‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ object SimpleFeatureSpec {
121121
protected def builderHook(builder: AttributeTypeBuilder): Unit = {}
122122
}
123123

124-
import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes.AttributeOptions.{OptDefault, OptIndex, OptSrid}
124+
import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes.AttributeOptions.{OptDefault, OptSrid}
125125

126126
private val simpleOptionPattern = Pattern.compile("[a-zA-Z0-9_]+")
127127

@@ -189,10 +189,9 @@ object SimpleFeatureSpec {
189189
}
190190

191191
// default geoms are indicated by the *
192-
// we don't allow attribute indexing for geometries
193-
override protected def specOptions: Map[String, String] = options - OptDefault - OptIndex
194-
override protected def configOptions: Map[String, String] = options - OptIndex
195-
override protected def descriptorOptions: Map[String, String] = options - OptIndex
192+
override protected def specOptions: Map[String, String] = options - OptDefault
193+
override protected def configOptions: Map[String, String] = options
194+
override protected def descriptorOptions: Map[String, String] = options
196195
}
197196

198197
/**

‎geomesa-utils-parent/geomesa-utils/src/test/scala/org/locationtech/geomesa/utils/geotools/SimpleFeatureTypesTest.scala‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class SimpleFeatureTypesTest extends Specification {
3434

3535
"SimpleFeatureTypes" should {
3636
"create an sft that" >> {
37-
val sft = SimpleFeatureTypes.createType("testing", "id:Integer,dtg:Date,*geom:Point:srid=4326:index=true")
37+
val sft = SimpleFeatureTypes.createType("testing", "id:Integer,dtg:Date,*geom:Point:srid=4326")
3838
"has name \'test\'" >> { sft.getTypeName mustEqual "testing" }
3939
"has three attributes" >> { sft.getAttributeCount must be_==(3) }
4040
"has an id attribute which is " >> {
@@ -46,9 +46,10 @@ class SimpleFeatureTypesTest extends Specification {
4646
val geomDescriptor = sft.getGeometryDescriptor
4747
geomDescriptor.getLocalName must be equalTo "geom"
4848
}
49-
"not include index flag for geometry" >> {
49+
"include index flag for geometry" >> {
50+
val sft = SimpleFeatureTypes.createType("testing", "id:Integer,dtg:Date,*geom:Point:srid=4326:index=true")
5051
val geomDescriptor = sft.getGeometryDescriptor
51-
geomDescriptor.getUserData.get("index") must beNull
52+
geomDescriptor.getUserData.get("index") mustEqual "true"
5253
}
5354
"encode an sft properly" >> {
5455
SimpleFeatureTypes.encodeType(sft) must be equalTo s"id:Integer,dtg:Date,*geom:Point:srid=4326"
@@ -139,7 +140,7 @@ class SimpleFeatureTypesTest extends Specification {
139140
}
140141

141142
"return the indexed attributes (not including the default geometry)" >> {
142-
val sft = SimpleFeatureTypes.createType("testing", "id:Integer:index=false,dtg:Date:index=true,*geom:Point:srid=4326:index=true")
143+
val sft = SimpleFeatureTypes.createType("testing", "id:Integer:index=false,dtg:Date:index=true,*geom:Point:srid=4326")
143144
val indexed = sft.getAttributeDescriptors.asScala.collect {
144145
case d if java.lang.Boolean.valueOf(d.getUserData.get(AttributeOptions.OptIndex).asInstanceOf[String]) => d.getLocalName
145146
}

0 commit comments

Comments
 (0)