Make webservice use query starter query logic factory xml files and the config service for properties

Author: billoley

docker/config/application-query.yml

@@ -10,6 +10,10 @@ warehouse:
     host: localhost
     port: 8125
   tables:
+    annotation:
+      name: 'datawave.annotation'
+    annotationSource:
+      name: 'datawave.annotationSource'
     shard:
       name: 'datawave.shard'
     index:
@@ -24,6 +28,8 @@ warehouse:
       name: 'datawave.metadata'
     edge:
       name: 'datawave.edge'
+    ssdeepIndex:
+      name: 'ssdeepIndex'
   errorTables:
     shard:
       name: "datawave.error_s"
@@ -36,7 +42,7 @@ warehouse:
     metadata:
       name: "datawave.error_m"
     model:
-      name: "datawave.error_m"
+      name: "datawave.metadata"
   metricTables:
     shard:
       name: "datawave.queryMetrics_s"
@@ -45,7 +51,7 @@ warehouse:
     reverseIndex:
       name: "datawave.queryMetrics_r"
     dateIndex:
-      name: ""
+      name: "datawave.queryMetrics_di"
     metadata:
       name: "datawave.queryMetrics_m"
     model:
@@ -58,8 +64,8 @@ warehouse:
     fullTableScanEnabled: false
     baseIteratorPriority: 100
     maxIndexScanTimeMillis: 31536000000
-    eventPerDayThreshold: 40000
-    shardsPerDayThreshold: 20
+    eventPerDayThreshold: 10000
+    shardsPerDayThreshold: 10
     initialMaxTermThreshold: 2000
     finalMaxTermThreshold: 2000
     maxDepthThreshold: 2000
@@ -72,6 +78,7 @@ warehouse:
     maxOrExpansionFstThreshold: 750
     maxFieldIndexRangeSplit: 16
     maxIvaratorSources: 20
+    maxIvaratorSourceWait: 1800000
     maxEvaluationPipelines: 16
     maxPipelineCachedResults: 16
     hdfsSiteConfigURLs: 'file://${HADOOP_CONF_DIR:/etc/hadoop/conf}/core-site.xml,file://${HADOOP_CONF_DIR:/etc/hadoop/conf}/hdfs-site.xml'
@@ -81,6 +88,7 @@ warehouse:
     ivaratorCacheScanPersistThreshold: 100000
     ivaratorCacheScanTimeoutMinutes: 60
     yieldThresholdMs: 188400
+    maxYields: 20
     modelName: 'DATAWAVE'
     edgeModelName: 'DATAWAVE_EDGE'
 
@@ -162,8 +170,8 @@ datawave:
           disableIndexOnlyDocuments: false
           indexOnlyFilterFunctionsEnabled: false
           includeHierarchyFields: false
-          hierarchyFieldOptions:
-            "FOO": "BAR"
+          #hierarchyFieldOptions:
+          #  "FOO": "BAR"
           baseIteratorPriority: ${warehouse.defaults.baseIteratorPriority}
           maxIndexScanTimeMillis: ${warehouse.defaults.maxIndexScanTimeMillis}
           collapseUids: false
@@ -177,10 +185,10 @@ datawave:
           enricherClassNames:
             - 'datawave.query.enrich.DatawaveTermFrequencyEnricher'
           useFilters: false
-          filterClassNames:
-            - 'foo.bar'
-          filterOptions:
-            'bar': "foo"
+          #filterClassNames:
+          #  - 'foo.bar'
+          #filterOptions:
+          #  'bar': "foo"
           auditType: "ACTIVE"
           logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed"
           eventPerDayThreshold: ${warehouse.defaults.eventPerDayThreshold}
@@ -195,18 +203,22 @@ datawave:
           maxOrExpansionFstThreshold: ${warehouse.defaults.maxOrExpansionFstThreshold}
           maxFieldIndexRangeSplit: ${warehouse.defaults.maxFieldIndexRangeSplit}
           maxIvaratorSources: ${warehouse.defaults.maxIvaratorSources}
+          maxIvaratorSourceWait: ${warehouse.defaults.maxIvaratorSourceWait}
           maxEvaluationPipelines: ${warehouse.defaults.maxEvaluationPipelines}
           maxPipelineCachedResults: ${warehouse.defaults.maxPipelineCachedResults}
           hdfsSiteConfigURLs: ${warehouse.defaults.hdfsSiteConfigURLs}
           zookeeperConfig: ${warehouse.accumulo.zookeepers}
           ivaratorCacheDirConfigs:
             - basePathURI: "hdfs://${HADOOP_HOST:localhost}:9000/IvaratorCache"
+          localIvaratorCacheDirConfigs:
+            - basePathURI: "file:///tmp/IvaratorCache"
           ivaratorFstHdfsBaseURIs: ${warehouse.defaults.ivaratorFstHdfsBaseURIs}
           ivaratorCacheBufferSize: ${warehouse.defaults.ivaratorCacheBufferSize}
           ivaratorMaxOpenFiles: ${warehouse.defaults.ivaratorMaxOpenFiles}
           ivaratorCacheScanPersistThreshold: ${warehouse.defaults.ivaratorCacheScanPersistThreshold}
           ivaratorCacheScanTimeoutMinutes: ${warehouse.defaults.ivaratorCacheScanTimeoutMinutes}
           yieldThresholdMs: ${warehouse.defaults.yieldThresholdMs}
+          maxYields: ${warehouse.defaults.maxYields}
           eventQueryDataDecoratorTransformer:
             requestedDecorators:
               - "CSV"
@@ -229,7 +241,7 @@ datawave:
           sendTimingToStatsd: false
           collectQueryMetrics: true
           collectTimingDetails: true
-          logTimingDetails: true
+          logTimingDetails: false
           statsdHost: ${warehouse.statsd.host}
           statsdPort: ${warehouse.statsd.port}
           evaluationOnlyFields: ""
@@ -435,7 +447,7 @@ datawave:
           useEnrichers: false
           auditType: "NONE"
           logicDescription: "Fast boolean query over indexed fields, only returning fields queried on"
-          eventPerDayThreshold: 40000
+          eventPerDayThreshold: 10000
           shardsPerDayThreshold: ${warehouse.defaults.shardsPerDayThreshold}
           initialMaxTermThreshold: ${warehouse.defaults.initialMaxTermThreshold}
           finalMaxTermThreshold: ${warehouse.defaults.finalMaxTermThreshold}
@@ -470,6 +482,52 @@ datawave:
           edgeModelName: "DATAWAVE_EDGE"
           modelTableName: ${warehouse.tables.model.name}
 
+        SSDeepSimilarityQuery:
+          checkpointable: ${warehouse.defaults.checkpointable}
+          tableName: ${warehouse.tables.ssdeepIndex.name}
+          maxResults: -1
+          maxWork: -1
+          auditType: "NONE"
+          indexBuckets: 32
+          queryThreads: 100
+          maxRepeatedCharacters: 3
+          bucketEncodingBase: 32
+          bucketEncodingLength: 2
+          logicDescription: "Query that will retrieve similar ssdeep hashes"
+
+        SSDeepDiscoveryQuery:
+          checkpointable: ${warehouse.defaults.checkpointable}
+          tableName: ${warehouse.tables.index.name}
+          indexTableName: ${warehouse.tables.index.name}
+          reverseIndexTableName: ${warehouse.tables.reverseIndex.name}
+          maxResults: -1
+          maxWork: -1
+          modelTableName: ${warehouse.tables.model.name}
+          modelName: ${warehouse.defaults.modelName}
+          fullTableScanEnabled: ${warehouse.defaults.fullTableScanEnabled}
+          allowLeadingWildcard: true
+          auditType: "NONE"
+          logicDescription: "Discovery query that returns information from the index about the supplied term(s)"
+
+        SSDeepSimilarityDiscoveryQuery:
+          checkpointable: ${warehouse.defaults.checkpointable}
+          tableName: ${warehouse.tables.ssdeepIndex.name}
+          auditType: "NONE"
+          logicDescription: "Query that will retrieve similar ssdeep hashes and then automatically perform a follow-on Discovery query that returns information from the index about the similar hashes"
+
+        KeywordQuery:
+          tableName: ${warehouse.tables.shard.name}
+          maxResults: -1
+          maxWork: -1
+          auditType: "NONE"
+          logicDescription: "Query that returns keywords from content"
+          queryThreads: 100
+          maxContentChars: 32768
+          maxKeywords: 10
+          maxNgrams: 3
+          minNgrams: 1
+          maxScore: "0.6f"
+
     uuidTypes: &defaultUuidTypes
       'EVENT_ID':
         fieldName: 'EVENT_ID'
@@ -496,12 +554,12 @@ datawave:
     lookup:
       columnVisibility: ""
       beginDate: "20100101 000000.000"
-      types: *defaultUuidTypes
+      uuidTypes: *defaultUuidTypes
 
     translateid:
       columnVisibility: ""
       beginDate: "20100101 000000.000"
-      types: *defaultUuidTypes
+      uuidTypes: *defaultUuidTypes
 
     edge:
       # Uncomment the following line to override the edge beans to load

docker/config/wildfly.yml

@@ -0,0 +1,137 @@
+logging:
+  level:
+    datawave: info
+    org.springframework: info
+
+accumulo:
+  zookeepers: 'localhost:2181'
+  instanceName: 'accumulo'
+  username: 'root'
+  password: 'secret'
+
+query:
+  page:
+    shortCircuit:
+      minutes: 55
+    size:
+      default: 10
+      max: 10000   
+      shortCircuit:
+        minutes: 30
+    byte:
+      trigger: 0
+  maxCallTime:
+    minutes: 60
+  expiration:
+    minutes: 15
+    
+spring.context.debug.dir: "${java.io.tmpdir}/contextDebug-${random.uuid}"
+
+datawave:
+  cachedresults:
+    hdfsUri: 'hdfs://localhost:8020/'
+    exportDir: '/CachedResults'
+    rowsPerBatch: 10
+    daysToLive: 1
+  configuration:
+    spring:
+      # if false, then the cdi.bean.context property will be used to find the XML file (default beanRefContext.xml)
+      # to construct the application context and these datawave.configuration.spring properties will be ignored 
+      configure-from-properties: true
+      ignoreUnresolvablePlaceholders: false
+      scan-base-packages:
+        - "datawave.webservice.config"
+        - "datawave.microservice.config.marking"
+        - "datawave.microservice.query.edge.config"
+        - "datawave.microservice.query.logic.config"
+      sources:
+        - "classpath*:datawave/annotation/AnnotationManagerFactory.xml"
+        - "classpath*:datawave/configuration/spring/CDIBeanPostProcessor.xml"
+        - "classpath*:datawave/marking/MarkingFunctionsContext.xml"
+        - "classpath*:datawave/metadata/MetadataHelperContext.xml"
+        - "classpath*:datawave/metadata/MetadataHelperCacheListenerContext.xml"
+        - "classpath*:datawave/modification/ModificationServices.xml"
+        - "classpath*:datawave/query/QueryExpiration.xml"
+        - "classpath*:datawave/query/CachedResults*.xml"
+        - "classpath*:datawave/security/PrincipalFactory.xml"
+        - "classpath*:datawave/security/TestDatawaveUserServiceConfiguration.xml"
+        - "classpath*:org/apache/accumulo/operations/LookupBean.xml"
+        - "classpath*:CacheContext.xml"
+        - "classpath*:QueryLogicFactory.xml"
+        - "classpath*:EdgeQueryLogicFactory.xml"
+        - "classpath*:KeywordExtractionQueryLogicFactory.xml"
+        - "classpath*:SSDeepQueryLogicFactory.xml"
+  query:
+    expiration:
+      minutes: ${query.expiration.minutes}
+      callTimeout: ${query.maxCallTime.minutes}
+      shortCircuitCheckTime: ${query.page.size.shortCircuit.minutes}
+      shortCircuitTimeout: ${query.page.shortCircuit.minutes}
+    logic:
+      factory:
+        enabled: true
+        importResources: false  
+      logics:
+        RemoteEventQuery:
+          tableName: ${warehouse.tables.shard.name}
+          maxResults: -1
+          auditType: "NONE"
+          logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed"
+    metrics:
+      remoteProcessorThreads: 4
+      batchSize: 100
+      maxQueueSize: 250000
+      maxLatencyMs: 5000
+      maxShutdownMs: 30000
+      useRemoteService: false
+      timely:
+        host: "localhost"
+        port: 4242
+        metricFieldTags:
+          - "USER"
+          - "HOST"
+          - "QUERY_ID"
+          - "QUERY_LOGIC"
+    RemoteUserOperations:
+      scheme: "https"
+      host: "localhost"
+      port: 8443
+      uri: "/DataWave/Security/User/"
+    RemoteQueryService:
+      scheme: "https"
+      host: "query"
+      port: 8443
+      uri: "/query/v1/"
+  modification:
+    data:
+      tableName: ${warehouse.tables.metadata.name}
+      poolName: ${datawave.connection.factory.defaultPool}
+    handlers:
+      requiresAudit: false
+      authorizedRoles:
+        - "AuthorizedUser"
+      eventTableName: ${warehouse.tables.shard.name}
+      metadataTableName: ${warehouse.tables.metadata.name}
+      indexTableName: ${warehouse.tables.index.name}
+      reverseIndexTableName: ${warehouse.tables.reverseIndex.name}
+      securityMarkingExemptFields:
+        - "exemptField1"
+      indexOnlyMap:
+        key1: "value1"
+      indexOnlySuffixes:
+        - "suffix1"
+      contentFields:
+        - "contentField1"
+  table:
+    cache:
+      enabled: false
+      zookeepers: ${accumulo.zookeepers}
+      poolName: ${datawave.connection.factory.defaultPool}
+      reloadInterval: 86400000
+      evictionReaperIntervalInSeconds: 30
+      numLocks: 300
+      maxRetries: 10
+      tableNames:
+        - datawave.metadata
+        - datawave.queryMetrics_m
+        - datawave.error_m

docs/pom.xml

@@ -162,7 +162,7 @@
         <dependency>
             <groupId>org.springframework.security</groupId>
             <artifactId>spring-security-core</artifactId>
-            <version>${version.springframework}</version>
+            <version>${version.spring-security-core}</version>
         </dependency>
         <dependency>
             <groupId>org.wildfly</groupId>
@@ -318,7 +318,7 @@
                         <configuration combine.children="append">
                             <sourcepath>${project.build.directory}/dependency</sourcepath>
                             <outputDirectory>${project.build.directory}/apidocs</outputDirectory>
-                            <excludePackageNames>generated-sources.*:datawave.microservice.querymetric</excludePackageNames>
+                            <excludePackageNames>generated-sources.*:datawave.microservice.querymetric:datawave.autoconfigure</excludePackageNames>
                         </configuration>
                     </execution>
                 </executions>

microservices/configcheck/pom.xml

@@ -339,8 +339,8 @@
                     <threadCountClasses>1</threadCountClasses>
                     <threadCountMethods>0</threadCountMethods>
                     <threadCountSuites>0</threadCountSuites>
-                    <argLine>@{argLine} -XX:+TieredCompilation -XX:TieredStopAtLevel=1  -Dfile.encoding=UTF8 -Duser.timezone=GMT -Xmx1024m -Dapple.awt.UIElement=true -Djava.security.krb5.realm= -Djava.security.krb5.kdc=</argLine>
-                    <redirectTestOutputToFile>true</redirectTestOutputToFile>
+                    <argLine>-XX:+TieredCompilation -XX:TieredStopAtLevel=1  -Dfile.encoding=UTF8 -Duser.timezone=GMT -Xmx1024m -Dapple.awt.UIElement=true -Djava.security.krb5.realm= -Djava.security.krb5.kdc=</argLine>
+                    <redirectTestOutputToFile>false</redirectTestOutputToFile>
                     <runOrder>random</runOrder>
                 </configuration>
             </plugin>