diff --git a/build.gradle b/build.gradle index 78dcefc1b80..631e8ac706f 100644 --- a/build.gradle +++ b/build.gradle @@ -179,6 +179,7 @@ ext.libs = [ commons_lang : "org.apache.commons:commons-lang3", // version controlled by dropwizard-dependencies commons_pool2 : "org.apache.commons:commons-pool2:2.12.0", commons_text : "org.apache.commons:commons-text", // version controlled by dropwizard-dependencies + commons_collections4 : "org.apache.commons:commons-collections4:4.4", // Not in dropwizard-dependencies classgraph : "io.github.classgraph:classgraph:4.8.146", curator_client : "org.apache.curator:curator-client:$versions.curator", curator_framework : "org.apache.curator:curator-framework:$versions.curator", diff --git a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticDataStores.java b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticDataStores.java index fd1d4c08077..0a6f5d64567 100644 --- a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticDataStores.java +++ b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticDataStores.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.analytics.impl; import stroom.analytics.rule.impl.AnalyticRuleStore; @@ -47,6 +63,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import stroom.view.shared.ViewDoc; import jakarta.inject.Inject; @@ -248,7 +265,7 @@ private LmdbDataStore createStore(final SearchRequest searchRequest) { final FieldIndex fieldIndex = new FieldIndex(); // Create a parameter map. - final Map paramMap = ParamUtil.createParamMap(searchRequest.getQuery().getParams()); + final Map paramMap = ParamUtil.createParamMap(searchRequest.getQuery().getParams()); // Create error consumer. final ErrorConsumer errorConsumer = new ErrorConsumerImpl(); @@ -274,7 +291,7 @@ private LmdbDataStore createAnalyticLmdbDataStore(final QueryKey queryKey, final TableSettings tableSettings, final ExpressionContext expressionContext, final FieldIndex fieldIndex, - final Map paramMap, + final Map paramMap, final DataStoreSettings dataStoreSettings, final ErrorConsumer errorConsumer) { final AnalyticResultStoreConfig storeConfig = analyticStoreConfigProvider.get(); diff --git a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticFields.java b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticFields.java index 8dd6331b72c..64dc646dfc8 100644 --- a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticFields.java +++ b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticFields.java @@ -18,6 +18,8 @@ import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import java.util.ArrayList; import java.util.List; @@ -34,35 +36,28 @@ public class AnalyticFields { .name(ANALYTICS_STORE_TYPE) .build(); - public static final String NAME = "Name"; - public static final String UUID = "UUID"; - public static final String TIME = "Time"; - public static final String VALUE = "Value"; - private static final List FIELDS = new ArrayList<>(); - private static final Map FIELD_MAP; - // Times - public static final QueryField TIME_FIELD = QueryField.createDate(TIME); - - public static final QueryField NAME_FIELD = QueryField.createText(NAME); - public static final QueryField UUID_FIELD = QueryField.createText(UUID); - public static final QueryField VALUE_FIELD = QueryField.createText(VALUE); - - static { - FIELDS.add(TIME_FIELD); - FIELDS.add(NAME_FIELD); - FIELDS.add(UUID_FIELD); - FIELDS.add(VALUE_FIELD); - - FIELD_MAP = FIELDS.stream() - .collect(Collectors.toMap(QueryField::getFldName, Function.identity())); - } + public static final QueryField TIME_FIELD = QueryField.createDate(CIKeys.TIME, true); + public static final QueryField NAME_FIELD = QueryField.createText(CIKeys.NAME, true); + public static final QueryField UUID_FIELD = QueryField.createText(CIKeys.UUID, true); + public static final QueryField VALUE_FIELD = QueryField.createText(CIKeys.VALUE, true); + + private static final List FIELDS = List.of( + TIME_FIELD, + NAME_FIELD, + UUID_FIELD, + VALUE_FIELD); + + private static final Map FIELD_NAME_TO_FIELD_MAP = FIELDS.stream() + .collect(Collectors.toMap( + QueryField::getFldNameAsCIKey, + Function.identity())); public static List getFields() { return new ArrayList<>(FIELDS); } - public static Map getFieldMap() { - return FIELD_MAP; + public static Map getFieldMap() { + return FIELD_NAME_TO_FIELD_MAP; } } diff --git a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticsNodeSearchTaskHandler.java b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticsNodeSearchTaskHandler.java index 386f70c87bb..a66bd3c83a8 100644 --- a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticsNodeSearchTaskHandler.java +++ b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/AnalyticsNodeSearchTaskHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.analytics.impl; @@ -62,6 +61,7 @@ import stroom.util.concurrent.UncheckedInterruptedException; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; @@ -148,10 +148,10 @@ private void doSearch(final TaskContext parentContext, final List> futures = new ArrayList<>(); try { final FieldIndex fieldIndex = coprocessors.getFieldIndex(); - final Map fieldMap = AnalyticFields.getFieldMap(); + final Map fieldMap = AnalyticFields.getFieldMap(); final QueryField[] fieldArray = new QueryField[fieldIndex.size()]; for (int i = 0; i < fieldArray.length; i++) { - final String fieldName = fieldIndex.getField(i); + final CIKey fieldName = fieldIndex.getFieldAsCIKey(i); final QueryField field = fieldMap.get(fieldName); if (field == null) { throw new RuntimeException("Field '" + fieldName + "' is not valid for this datasource"); @@ -331,7 +331,7 @@ public TableResultConsumer errors(final List errors) { public TableResultConsumer columns(final List columns) { this.columns = columns; fieldIndex = new FieldIndex(); - columns.forEach(column -> fieldIndex.create(column.getName())); + columns.forEach(column -> fieldIndex.create(column.getNameAsCIKey())); return this; } @@ -357,11 +357,11 @@ public TableResultConsumer addRow(final Row row) { } final String value = sb.toString(); - final Map attributeMap = new HashMap<>(); - attributeMap.put(AnalyticFields.NAME_FIELD.getFldName(), analyticRuleDoc.getName()); - attributeMap.put(AnalyticFields.UUID_FIELD.getFldName(), analyticRuleDoc.getUuid()); - attributeMap.put(AnalyticFields.TIME_FIELD.getFldName(), time); - attributeMap.put(AnalyticFields.VALUE_FIELD.getFldName(), value); + final Map attributeMap = new HashMap<>(); + attributeMap.put(AnalyticFields.NAME_FIELD.getFldNameAsCIKey(), analyticRuleDoc.getName()); + attributeMap.put(AnalyticFields.UUID_FIELD.getFldNameAsCIKey(), analyticRuleDoc.getUuid()); + attributeMap.put(AnalyticFields.TIME_FIELD.getFldNameAsCIKey(), time); + attributeMap.put(AnalyticFields.VALUE_FIELD.getFldNameAsCIKey(), value); if (expressionMatcher.match(attributeMap, expression)) { hitCount.increment(); diff --git a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/DetectionConsumerProxy.java b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/DetectionConsumerProxy.java index 9412e01dc2f..5472f1d92df 100644 --- a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/DetectionConsumerProxy.java +++ b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/DetectionConsumerProxy.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.analytics.impl; import stroom.analytics.shared.AnalyticRuleDoc; @@ -9,7 +25,6 @@ import stroom.query.common.v2.CompiledColumns; import stroom.query.common.v2.format.ColumnFormatter; import stroom.query.common.v2.format.FormatterFactory; -import stroom.query.language.functions.FieldIndex; import stroom.query.language.functions.Generator; import stroom.query.language.functions.Val; import stroom.query.language.functions.ValuesConsumer; @@ -18,6 +33,7 @@ import stroom.util.NullSafe; import stroom.util.date.DateUtil; import stroom.util.shared.Severity; +import stroom.util.shared.query.FieldNames; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -198,9 +214,9 @@ private void writeRecord(final ColumnValue[] columnValues) { NullSafe.consume(columnValue, ColumnValue::val, val -> { final Column column = columnValue.column(); final String columnName = column.getName(); - if (FieldIndex.isStreamIdFieldName(columnName)) { + if (FieldNames.isStreamIdFieldName(columnName)) { streamId.set(getSafeLong(val)); - } else if (FieldIndex.isEventIdFieldName(columnName)) { + } else if (FieldNames.isEventIdFieldName(columnName)) { eventId.set(getSafeLong(val)); } else { final String fieldValStr = fieldFormatter.format(column, val); diff --git a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/ScheduledQueryAnalyticExecutor.java b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/ScheduledQueryAnalyticExecutor.java index 2eee1e071b4..b93e6848bd3 100644 --- a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/ScheduledQueryAnalyticExecutor.java +++ b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/ScheduledQueryAnalyticExecutor.java @@ -76,6 +76,7 @@ import stroom.util.shared.ResultPage; import stroom.util.shared.Severity; import stroom.util.shared.scheduler.Schedule; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -208,7 +209,7 @@ private void execAnalytic(final AnalyticRuleDoc analytic, .builder() .ownerDocRef(analytic.asDocRef()) .enabled(true) - .nodeName(StringMatch.equals(nodeInfo.getThisNodeName(), true)) + .nodeName(StringMatch.equalsIgnoreCase(nodeInfo.getThisNodeName())) .build(); final ResultPage executionSchedules = executionScheduleDao.fetchExecutionSchedule(request); @@ -366,7 +367,7 @@ private boolean process(final AnalyticRuleDoc analytic, // Now consume all results as detections. final TableSettings tableSettings = resultRequest.getMappings().getFirst(); final List columns = tableSettings.getColumns(); - final Map paramMap = ParamUtil + final Map paramMap = ParamUtil .createParamMap(mappedRequest.getQuery().getParams()); final CompiledColumns compiledColumns = CompiledColumns.create( expressionContext, diff --git a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/StreamingAnalyticProcessorTaskDecorator.java b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/StreamingAnalyticProcessorTaskDecorator.java index 9deb741b75d..2ca1288f62b 100644 --- a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/StreamingAnalyticProcessorTaskDecorator.java +++ b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/StreamingAnalyticProcessorTaskDecorator.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.analytics.impl; @@ -41,6 +40,7 @@ import stroom.util.concurrent.UncheckedInterruptedException; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -116,7 +116,7 @@ private Optional createEventConsumer(final StreamingA final ExpressionContext expressionContext = expressionContextFactory .createContext(searchRequest); final TableSettings tableSettings = searchRequest.getResultRequests().getFirst().getMappings().getFirst(); - final Map paramMap = ParamUtil.createParamMap(searchRequest.getQuery().getParams()); + final Map paramMap = ParamUtil.createParamMap(searchRequest.getQuery().getParams()); final CompiledColumns compiledColumns = CompiledColumns.create(expressionContext, tableSettings.getColumns(), paramMap); @@ -135,6 +135,7 @@ private Optional createEventConsumer(final StreamingA try { final Provider detectionConsumerProvider = detectionConsumerFactory.create(analytic.analyticRuleDoc()); + detectionConsumerProxy.setAnalyticRuleDoc(analytic.analyticRuleDoc()); detectionConsumerProxy.setCompiledColumns(compiledColumns); detectionConsumerProxy.setDetectionsConsumerProvider(detectionConsumerProvider); @@ -162,6 +163,10 @@ private Optional createEventConsumer(final StreamingA } } + + // -------------------------------------------------------------------------------- + + private static class NullFieldListConsumer implements AnalyticFieldListConsumer { @Override diff --git a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/TableBuilderAnalyticExecutor.java b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/TableBuilderAnalyticExecutor.java index 32ad7325812..2e62e94df68 100644 --- a/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/TableBuilderAnalyticExecutor.java +++ b/stroom-analytics/stroom-analytics-impl/src/main/java/stroom/analytics/impl/TableBuilderAnalyticExecutor.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.analytics.impl; import stroom.analytics.impl.AnalyticDataStores.AnalyticDataStore; @@ -61,6 +77,7 @@ import stroom.util.logging.LogExecutionTime; import stroom.util.logging.LogUtil; import stroom.util.shared.UserRef; +import stroom.util.shared.string.CIKey; import stroom.util.shared.time.SimpleDuration; import stroom.util.shared.time.TimeUnit; import stroom.util.time.SimpleDurationUtil; @@ -362,7 +379,7 @@ private void processStream(final DocRef pipelineDocRef, final List analytics, final Meta meta, final TaskContext parentTaskContext) { - final Map metaAttributeMap = MetaAttributeMapUtil + final Map metaAttributeMap = MetaAttributeMapUtil .createAttributeMap(meta); final List fieldListConsumers = new ArrayList<>(); @@ -477,7 +494,7 @@ private AnalyticFieldListConsumer createLmdbConsumer(final TableBuilderAnalytic private boolean ignoreStream(final TableBuilderAnalytic analytic, final Meta meta, - final Map metaAttributeMap) { + final Map metaAttributeMap) { final TableBuilderAnalyticTrackerData trackerData = analytic.trackerData; final long minStreamId = trackerData.getMinStreamId(); final long minCreateTime = diff --git a/stroom-analytics/stroom-analytics-rule-impl/src/main/java/stroom/analytics/rule/impl/AnalyticRuleStoreImpl.java b/stroom-analytics/stroom-analytics-rule-impl/src/main/java/stroom/analytics/rule/impl/AnalyticRuleStoreImpl.java index 6d145258b61..3defd383d22 100644 --- a/stroom-analytics/stroom-analytics-rule-impl/src/main/java/stroom/analytics/rule/impl/AnalyticRuleStoreImpl.java +++ b/stroom-analytics/stroom-analytics-rule-impl/src/main/java/stroom/analytics/rule/impl/AnalyticRuleStoreImpl.java @@ -304,8 +304,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-annotation/stroom-annotation-api/build.gradle b/stroom-annotation/stroom-annotation-api/build.gradle index 87f2bb18701..cbe5411315a 100644 --- a/stroom-annotation/stroom-annotation-api/build.gradle +++ b/stroom-annotation/stroom-annotation-api/build.gradle @@ -4,6 +4,7 @@ dependencies { implementation project(':stroom-core-shared') implementation project(':stroom-docref') implementation project(':stroom-query:stroom-query-api') + implementation project(':stroom-util-shared') implementation libs.jackson_annotations } diff --git a/stroom-annotation/stroom-annotation-api/src/main/java/stroom/annotation/api/AnnotationFields.java b/stroom-annotation/stroom-annotation-api/src/main/java/stroom/annotation/api/AnnotationFields.java index 9e017ccea90..5e57d3ac27d 100644 --- a/stroom-annotation/stroom-annotation-api/src/main/java/stroom/annotation/api/AnnotationFields.java +++ b/stroom-annotation/stroom-annotation-api/src/main/java/stroom/annotation/api/AnnotationFields.java @@ -1,6 +1,24 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.annotation.api; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import java.util.Arrays; import java.util.List; @@ -15,30 +33,31 @@ public interface AnnotationFields { String NAMESPACE = "annotation"; String ANNOTATION_FIELD_PREFIX = NAMESPACE + ":"; String ID = ANNOTATION_FIELD_PREFIX + "Id"; - String CREATED_ON = ANNOTATION_FIELD_PREFIX + "CreatedOn"; - String CREATED_BY = ANNOTATION_FIELD_PREFIX + "CreatedBy"; - String UPDATED_ON = ANNOTATION_FIELD_PREFIX + "UpdatedOn"; - String UPDATED_BY = ANNOTATION_FIELD_PREFIX + "UpdatedBy"; - String TITLE = ANNOTATION_FIELD_PREFIX + "Title"; - String SUBJECT = ANNOTATION_FIELD_PREFIX + "Subject"; - String STATUS = ANNOTATION_FIELD_PREFIX + "Status"; - String ASSIGNED_TO = ANNOTATION_FIELD_PREFIX + "AssignedTo"; - String COMMENT = ANNOTATION_FIELD_PREFIX + "Comment"; - String HISTORY = ANNOTATION_FIELD_PREFIX + "History"; - - QueryField ID_FIELD = QueryField.createId(ID); + +// String CREATED_ON = ANNOTATION_FIELD_PREFIX + "CreatedOn"; +// String CREATED_BY = ANNOTATION_FIELD_PREFIX + "CreatedBy"; +// String UPDATED_ON = ANNOTATION_FIELD_PREFIX + "UpdatedOn"; +// String UPDATED_BY = ANNOTATION_FIELD_PREFIX + "UpdatedBy"; +// String TITLE = ANNOTATION_FIELD_PREFIX + "Title"; +// String SUBJECT = ANNOTATION_FIELD_PREFIX + "Subject"; +// String STATUS = ANNOTATION_FIELD_PREFIX + "Status"; +// String ASSIGNED_TO = ANNOTATION_FIELD_PREFIX + "AssignedTo"; +// String COMMENT = ANNOTATION_FIELD_PREFIX + "Comment"; +// String HISTORY = ANNOTATION_FIELD_PREFIX + "History"; + + QueryField ID_FIELD = QueryField.createId(CIKeys.ANNO_ID, true); // AbstractField STREAM_ID_FIELD = QueryField.createId(IndexConstants.STREAM_ID); // AbstractField EVENT_ID_FIELD = QueryField.createId(IndexConstants.EVENT_ID); - QueryField CREATED_ON_FIELD = QueryField.createDate(CREATED_ON); - QueryField CREATED_BY_FIELD = QueryField.createText(CREATED_BY); - QueryField UPDATED_ON_FIELD = QueryField.createDate(UPDATED_ON); - QueryField UPDATED_BY_FIELD = QueryField.createText(UPDATED_BY); - QueryField TITLE_FIELD = QueryField.createText(TITLE); - QueryField SUBJECT_FIELD = QueryField.createText(SUBJECT); - QueryField STATUS_FIELD = QueryField.createText(STATUS); - QueryField ASSIGNED_TO_FIELD = QueryField.createText(ASSIGNED_TO); - QueryField COMMENT_FIELD = QueryField.createText(COMMENT); - QueryField HISTORY_FIELD = QueryField.createText(HISTORY); + QueryField CREATED_ON_FIELD = QueryField.createDate(CIKeys.ANNO_CREATED_ON, true); + QueryField CREATED_BY_FIELD = QueryField.createText(CIKeys.ANNO_CREATED_BY, true); + QueryField UPDATED_ON_FIELD = QueryField.createDate(CIKeys.ANNO_UPDATED_ON, true); + QueryField UPDATED_BY_FIELD = QueryField.createText(CIKeys.ANNO_UPDATED_BY, true); + QueryField TITLE_FIELD = QueryField.createText(CIKeys.ANNO_TITLE, true); + QueryField SUBJECT_FIELD = QueryField.createText(CIKeys.ANNO_SUBJECT, true); + QueryField STATUS_FIELD = QueryField.createText(CIKeys.ANNO_STATUS, true); + QueryField ASSIGNED_TO_FIELD = QueryField.createText(CIKeys.ANNO_ASSIGNED_TO, true); + QueryField COMMENT_FIELD = QueryField.createText(CIKeys.ANNO_COMMENT, true); + QueryField HISTORY_FIELD = QueryField.createText(CIKeys.ANNO_HISTORY, true); List FIELDS = Arrays.asList( ID_FIELD, @@ -54,6 +73,9 @@ public interface AnnotationFields { ASSIGNED_TO_FIELD, COMMENT_FIELD, HISTORY_FIELD); - Map FIELD_MAP = FIELDS.stream() - .collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + + Map FIELD_MAP = FIELDS.stream() + .collect(Collectors.toMap( + QueryField::getFldNameAsCIKey, + Function.identity())); } diff --git a/stroom-annotation/stroom-annotation-impl-db/src/main/java/stroom/annotation/impl/db/AnnotationDaoImpl.java b/stroom-annotation/stroom-annotation-impl-db/src/main/java/stroom/annotation/impl/db/AnnotationDaoImpl.java index 210ba1ef7aa..be8e8956026 100644 --- a/stroom-annotation/stroom-annotation-impl-db/src/main/java/stroom/annotation/impl/db/AnnotationDaoImpl.java +++ b/stroom-annotation/stroom-annotation-impl-db/src/main/java/stroom/annotation/impl/db/AnnotationDaoImpl.java @@ -50,6 +50,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.shared.UserRef; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import org.jooq.Condition; @@ -103,11 +104,11 @@ private ExpressionMapper createExpressionMapper(final ExpressionMapperFactory ex // expressionMapper.map(AnnotationDataSource.EVENT_ID_FIELD, ANNOTATION_DATA_LINK.EVENT_ID, Long::valueOf); expressionMapper.map(AnnotationFields.CREATED_ON_FIELD, ANNOTATION.CREATE_TIME_MS, - value -> DateExpressionParser.getMs(AnnotationFields.CREATED_ON, value)); + value -> DateExpressionParser.getMs(AnnotationFields.CREATED_ON_FIELD, value)); expressionMapper.map(AnnotationFields.CREATED_BY_FIELD, ANNOTATION.CREATE_USER, value -> value); expressionMapper.map(AnnotationFields.UPDATED_ON_FIELD, ANNOTATION.UPDATE_TIME_MS, - value -> DateExpressionParser.getMs(AnnotationFields.UPDATED_ON, value)); + value -> DateExpressionParser.getMs(AnnotationFields.UPDATED_ON_FIELD, value)); expressionMapper.map(AnnotationFields.UPDATED_BY_FIELD, ANNOTATION.UPDATE_USER, value -> value); expressionMapper.map(AnnotationFields.TITLE_FIELD, ANNOTATION.TITLE, value -> value); expressionMapper.map(AnnotationFields.SUBJECT_FIELD, ANNOTATION.SUBJECT, value -> value); @@ -504,7 +505,8 @@ private void changeField(final long annotationId, public void search(final ExpressionCriteria criteria, final FieldIndex fieldIndex, final ValuesConsumer consumer) { - final String[] fieldNames = fieldIndex.getFields(); + final List fieldNames = fieldIndex.getFieldsAsCIKeys(); + final int fieldCount = fieldNames.size(); final Condition condition = createCondition(criteria.getExpression()); final List> dbFields = new ArrayList<>(valueMapper.getDbFieldsByName(fieldNames)); final Mapper[] mappers = valueMapper.getMappersForFieldNames(fieldNames); @@ -519,13 +521,13 @@ public void search(final ExpressionCriteria criteria, while (cursor.hasNext()) { final Result result = cursor.fetchNext(1000); - result.forEach(r -> { - final Val[] arr = new Val[fieldNames.length]; - for (int i = 0; i < fieldNames.length; i++) { + result.forEach(rec -> { + final Val[] arr = new Val[fieldCount]; + for (int i = 0; i < fieldCount; i++) { Val val = ValNull.INSTANCE; final Mapper mapper = mappers[i]; if (mapper != null) { - val = mapper.map(r); + val = mapper.map(rec); } arr[i] = val; } diff --git a/stroom-annotation/stroom-annotation-impl/src/main/java/stroom/annotation/impl/AnnotationReceiverDecoratorFactory.java b/stroom-annotation/stroom-annotation-impl/src/main/java/stroom/annotation/impl/AnnotationReceiverDecoratorFactory.java index 6b71d6bfdd6..3bcb2ac466b 100644 --- a/stroom-annotation/stroom-annotation-impl/src/main/java/stroom/annotation/impl/AnnotationReceiverDecoratorFactory.java +++ b/stroom-annotation/stroom-annotation-impl/src/main/java/stroom/annotation/impl/AnnotationReceiverDecoratorFactory.java @@ -18,6 +18,7 @@ import stroom.annotation.api.AnnotationFields; import stroom.annotation.shared.Annotation; +import stroom.datasource.api.v2.QueryField; import stroom.expression.matcher.ExpressionMatcher; import stroom.expression.matcher.ExpressionMatcherFactory; import stroom.index.shared.IndexConstants; @@ -26,6 +27,7 @@ import stroom.query.api.v2.Query; import stroom.query.language.functions.FieldIndex; import stroom.query.language.functions.Val; +import stroom.query.language.functions.ValDate; import stroom.query.language.functions.ValLong; import stroom.query.language.functions.ValNull; import stroom.query.language.functions.ValString; @@ -37,6 +39,8 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.shared.UserRef; +import stroom.util.shared.string.CIHashSet; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; @@ -55,29 +59,35 @@ class AnnotationReceiverDecoratorFactory implements AnnotationsDecoratorFactory private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(AnnotationReceiverDecoratorFactory.class); - private static final Map> VALUE_MAPPING = Map.ofEntries( - nullSafeEntry(AnnotationFields.ID, Annotation::getId), - nullSafeEntry(AnnotationFields.TITLE, Annotation::getTitle), - nullSafeEntry(AnnotationFields.SUBJECT, Annotation::getSubject), - nullSafeEntry(AnnotationFields.STATUS, Annotation::getStatus), - nullSafeEntry(AnnotationFields.ASSIGNED_TO, annotation -> + private static final Map> VALUE_MAPPING = Map.ofEntries( + nullSafeEntry(AnnotationFields.ID_FIELD, Annotation::getId), +// nullSafeEntry(AnnotationFields.CREATED_ON_FIELD, Annotation::getCreateTime, createTimeEpochMs -> +// Val.nullSafeCreate(createTimeEpochMs, ValDate::create)), +// nullSafeEntry(AnnotationFields.CREATED_BY_FIELD, Annotation::getCreateUser), +// nullSafeEntry(AnnotationFields.UPDATED_ON_FIELD, Annotation::getUpdateTime, updateTimeEpochMs -> +// Val.nullSafeCreate(updateTimeEpochMs, ValDate::create)), +// nullSafeEntry(AnnotationFields.UPDATED_BY_FIELD, Annotation::getUpdateUser), + nullSafeEntry(AnnotationFields.TITLE_FIELD, Annotation::getTitle), + nullSafeEntry(AnnotationFields.SUBJECT_FIELD, Annotation::getSubject), + nullSafeEntry(AnnotationFields.STATUS_FIELD, Annotation::getStatus), + nullSafeEntry(AnnotationFields.ASSIGNED_TO_FIELD, annotation -> NullSafe.get(annotation.getAssignedTo(), UserRef::toDisplayString)), - nullSafeEntry(AnnotationFields.COMMENT, Annotation::getComment), - nullSafeEntry(AnnotationFields.HISTORY, Annotation::getHistory)); - - private static final Map> OBJECT_MAPPING = Map.ofEntries( - Map.entry(AnnotationFields.ID, Annotation::getId), - Map.entry(AnnotationFields.CREATED_ON, Annotation::getCreateTime), - Map.entry(AnnotationFields.CREATED_BY, Annotation::getCreateUser), - Map.entry(AnnotationFields.UPDATED_ON, Annotation::getUpdateTime), - Map.entry(AnnotationFields.UPDATED_BY, Annotation::getUpdateUser), - Map.entry(AnnotationFields.TITLE, Annotation::getTitle), - Map.entry(AnnotationFields.SUBJECT, Annotation::getSubject), - Map.entry(AnnotationFields.STATUS, Annotation::getStatus), - Map.entry(AnnotationFields.ASSIGNED_TO, annotation -> + nullSafeEntry(AnnotationFields.COMMENT_FIELD, Annotation::getComment), + nullSafeEntry(AnnotationFields.HISTORY_FIELD, Annotation::getHistory)); + + private static final Map> OBJECT_MAPPING = Map.ofEntries( + Map.entry(AnnotationFields.ID_FIELD.getFldNameAsCIKey(), Annotation::getId), + Map.entry(AnnotationFields.CREATED_ON_FIELD.getFldNameAsCIKey(), Annotation::getCreateTime), + Map.entry(AnnotationFields.CREATED_BY_FIELD.getFldNameAsCIKey(), Annotation::getCreateUser), + Map.entry(AnnotationFields.UPDATED_ON_FIELD.getFldNameAsCIKey(), Annotation::getUpdateTime), + Map.entry(AnnotationFields.UPDATED_BY_FIELD.getFldNameAsCIKey(), Annotation::getUpdateUser), + Map.entry(AnnotationFields.TITLE_FIELD.getFldNameAsCIKey(), Annotation::getTitle), + Map.entry(AnnotationFields.SUBJECT_FIELD.getFldNameAsCIKey(), Annotation::getSubject), + Map.entry(AnnotationFields.STATUS_FIELD.getFldNameAsCIKey(), Annotation::getStatus), + Map.entry(AnnotationFields.ASSIGNED_TO_FIELD.getFldNameAsCIKey(), annotation -> NullSafe.get(annotation.getAssignedTo(), UserRef::toDisplayString)), - Map.entry(AnnotationFields.COMMENT, Annotation::getComment), - Map.entry(AnnotationFields.HISTORY, Annotation::getHistory)); + Map.entry(AnnotationFields.COMMENT_FIELD.getFldNameAsCIKey(), Annotation::getComment), + Map.entry(AnnotationFields.HISTORY_FIELD.getFldNameAsCIKey(), Annotation::getHistory)); private final AnnotationDao annotationDao; private final ExpressionMatcherFactory expressionMatcherFactory; @@ -99,7 +109,7 @@ class AnnotationReceiverDecoratorFactory implements AnnotationsDecoratorFactory public ValuesConsumer create(final ValuesConsumer valuesConsumer, final FieldIndex fieldIndex, final Query query) { - final Integer annotationIdIndex = fieldIndex.getPos(AnnotationFields.ID); + final Integer annotationIdIndex = fieldIndex.getPos(AnnotationFields.ID_FIELD.getFldNameAsCIKey()); final Integer streamIdIndex = fieldIndex.getPos(IndexConstants.STREAM_ID); final Integer eventIdIndex = fieldIndex.getPos(IndexConstants.EVENT_ID); @@ -110,10 +120,10 @@ public ValuesConsumer create(final ValuesConsumer valuesConsumer, // Do we need to filter based on annotation attributes? final Function filter = createFilter(query.getExpression()); - final Set usedFields = new HashSet<>(Set.of(fieldIndex.getFields())); + final Set usedFields = new HashSet<>(fieldIndex.getFieldsAsCIKeys()); usedFields.retainAll(AnnotationFields.FIELD_MAP.keySet()); - if (filter == null && usedFields.size() == 0) { + if (filter == null && usedFields.isEmpty()) { return valuesConsumer; } @@ -132,7 +142,7 @@ public ValuesConsumer create(final ValuesConsumer valuesConsumer, } } - if (annotations.size() == 0) { + if (annotations.isEmpty()) { final Long streamId = getLong(values, streamIdIndex); final Long eventId = getLong(values, eventIdIndex); if (streamId != null && eventId != null) { @@ -141,7 +151,7 @@ public ValuesConsumer create(final ValuesConsumer valuesConsumer, } } - if (annotations.size() == 0) { + if (annotations.isEmpty()) { annotations.add(defaultAnnotation); } @@ -155,7 +165,7 @@ public ValuesConsumer create(final ValuesConsumer valuesConsumer, copy = Arrays.copyOf(values, fieldIndex.size()); } - for (final String field : usedFields) { + for (final CIKey field : usedFields) { setValue(copy, fieldIndex, field, annotation); } @@ -185,18 +195,18 @@ private Function createFilter(final ExpressionOperator expr final ExpressionOperator filteredExpression = expressionFilter.copy(expression); final List expressionValues = ExpressionUtil.values(filteredExpression); - if (expressionValues == null || expressionValues.size() == 0) { + if (NullSafe.isEmptyCollection(expressionValues)) { return null; } - final Set usedFields = new HashSet<>(ExpressionUtil.fields(filteredExpression)); - if (usedFields.size() == 0) { + final Set usedFields = new CIHashSet(ExpressionUtil.fields(filteredExpression)); + if (usedFields.isEmpty()) { return null; } final ExpressionMatcher expressionMatcher = expressionMatcherFactory.create(AnnotationFields.FIELD_MAP); return annotation -> { - final Map attributeMap = new HashMap<>(); - for (final String field : usedFields) { + final Map attributeMap = new HashMap<>(); + for (final CIKey field : usedFields) { final Object value = OBJECT_MAPPING.get(field) .apply(annotation); attributeMap.put(field, value); @@ -218,7 +228,7 @@ private Long getLong(final Val[] values, final int index) { private void setValue(final Val[] values, final FieldIndex fieldIndex, - final String field, + final CIKey field, final Annotation annotation) { final Integer index = fieldIndex.getPos(field); if (index != null && values.length > index) { @@ -226,7 +236,8 @@ private void setValue(final Val[] values, if (values[index] == null) { final Val val; try { - val = VALUE_MAPPING.get(field).apply(annotation); + val = VALUE_MAPPING.get(field) + .apply(annotation); } catch (Exception e) { throw new RuntimeException(e); } @@ -235,22 +246,23 @@ private void setValue(final Val[] values, } } - private static Entry> nullSafeEntry( - final String fieldName, + private static Entry> nullSafeEntry( + final QueryField field, final Function getter) { - return nullSafeEntry(fieldName, getter, null); + return nullSafeEntry(field, getter, null); } /** * @param creator An explicit creator function to use rather than inferring it from the type. */ - private static Entry> nullSafeEntry( - final String fieldName, + private static Entry> nullSafeEntry( + final QueryField field, final Function getter, final Function creator) { - Objects.requireNonNull(fieldName); + Objects.requireNonNull(field); Objects.requireNonNull(getter); + final CIKey fieldName = field.getFldNameAsCIKey(); return Map.entry(fieldName, annotation -> { T value = getter.apply(annotation); @@ -259,10 +271,10 @@ private static Entry> nullSafeEntry( } else { if (creator != null) { return creator.apply(value); - } else if (value instanceof String) { - return ValString.create((String) value); - } else if (value instanceof Long) { - return ValLong.create((Long) value); + } else if (value instanceof String str) { + return ValString.create(str); + } else if (value instanceof Long aLong) { + return ValLong.create(aLong); } else if (value instanceof final UserRef userRef) { return ValString.create(userRef.toDisplayString()); } else { diff --git a/stroom-app/src/test/java/stroom/contentindex/TestLuceneContentIndex.java b/stroom-app/src/test/java/stroom/contentindex/TestLuceneContentIndex.java index c40887a34a3..f69de2036da 100644 --- a/stroom-app/src/test/java/stroom/contentindex/TestLuceneContentIndex.java +++ b/stroom-app/src/test/java/stroom/contentindex/TestLuceneContentIndex.java @@ -90,7 +90,7 @@ void setup() { @Test void testBasic() { - test(StringMatch.contains("LOCATION"), 4, "Location"); + test(StringMatch.containsIgnoreCase("LOCATION"), 4, "Location"); } @Test @@ -100,7 +100,7 @@ void testCaseSensitive() { @Test void testPartial() { - test(StringMatch.contains("TION"), 4, "tion"); + test(StringMatch.containsIgnoreCase("TION"), 4, "tion"); } @Test @@ -110,7 +110,7 @@ void testPartialCaseSensitive() { @Test void testRegex() { - test(StringMatch.regex("XML\\w*"), 6, "xml"); + test(StringMatch.regexIgnoreCase("XML\\w*"), 6, "xml"); } @Test diff --git a/stroom-app/src/test/java/stroom/importexport/TestJsonSerialisation.java b/stroom-app/src/test/java/stroom/importexport/TestJsonSerialisation.java index 839af415b20..6194b35659b 100644 --- a/stroom-app/src/test/java/stroom/importexport/TestJsonSerialisation.java +++ b/stroom-app/src/test/java/stroom/importexport/TestJsonSerialisation.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.importexport; import stroom.util.json.JsonUtil; @@ -609,7 +625,8 @@ private static void addClass(final Set> stroomClasses, final Class c private static void addFields(final Set> stroomClasses, final Class parentClazz) { final Field[] fields = parentClazz.getDeclaredFields(); for (final Field field : fields) { - if (!Modifier.isStatic(field.getModifiers())) { + if (!Modifier.isStatic(field.getModifiers()) + && !field.isAnnotationPresent(JsonIgnore.class)) { addType(stroomClasses, field.getGenericType()); // addClass(stroomClasses, field.getType(), field.getGenericType()); } diff --git a/stroom-app/src/test/java/stroom/pipeline/TestPipelineStoreImpl.java b/stroom-app/src/test/java/stroom/pipeline/TestPipelineStoreImpl.java index f40ca6ca377..d3c92df67ce 100644 --- a/stroom-app/src/test/java/stroom/pipeline/TestPipelineStoreImpl.java +++ b/stroom-app/src/test/java/stroom/pipeline/TestPipelineStoreImpl.java @@ -37,7 +37,7 @@ void setUp() { } @TestFactory - Stream findByName() { + Stream findByName_caseSense() { return TestUtil.buildDynamicTestStream() .withInputType(String.class) .withWrappedOutputType(new TypeLiteral>() { @@ -45,7 +45,7 @@ Stream findByName() { .withTestFunction(testCase -> { final String nameFilter = testCase.getInput(); // Need to sort to ensure predictable order for tests - return pipelineStore.findByName(nameFilter, true) + return pipelineStore.findByName(nameFilter, true, true) .stream() .sorted(Comparator.naturalOrder()) .collect(Collectors.toList()); @@ -62,6 +62,33 @@ Stream findByName() { .build(); } + @TestFactory + Stream findByName_caseInSense() { + return TestUtil.buildDynamicTestStream() + .withInputType(String.class) + .withWrappedOutputType(new TypeLiteral>() { + }) + .withTestFunction(testCase -> { + final String nameFilter = testCase.getInput(); + // Need to sort to ensure predictable order for tests + return pipelineStore.findByName(nameFilter, true, false) + .stream() + .sorted(Comparator.naturalOrder()) + .collect(Collectors.toList()); + }) + .withSimpleEqualityAssertion() + .addCase(null, Collections.emptyList()) + .addCase("Pipe", Collections.emptyList()) + .addCase(PIPE_1_NAME.toLowerCase(), List.of(pipe1)) + .addCase(PIPE_1_NAME.toUpperCase(), List.of(pipe1)) + .addCase(PIPE_1_NAME, List.of(pipe1)) + .addCase("Pipe*", List.of(pipe1, pipe2, pipe3)) + .addCase("*a", List.of(pipe1, pipe2, pipe3)) + .addCase("*2*", List.of(pipe2)) + .addCase("* *aa*", List.of(pipe2, pipe3)) + .build(); + } + @TestFactory Stream findByName_no_wild_cards() { return TestUtil.buildDynamicTestStream() @@ -71,7 +98,7 @@ Stream findByName_no_wild_cards() { .withTestFunction(testCase -> { final String nameFilter = testCase.getInput(); // Need to sort to ensure predictable order for tests - return pipelineStore.findByName(nameFilter, false) + return pipelineStore.findByName(nameFilter, false, true) .stream() .sorted(Comparator.naturalOrder()) .collect(Collectors.toList()); diff --git a/stroom-app/src/test/java/stroom/receive/TestBaseModule.java b/stroom-app/src/test/java/stroom/receive/TestBaseModule.java index 36c0f1cb28e..3d67acc1e9a 100644 --- a/stroom-app/src/test/java/stroom/receive/TestBaseModule.java +++ b/stroom-app/src/test/java/stroom/receive/TestBaseModule.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.receive; import stroom.cache.impl.CacheModule; @@ -78,11 +94,6 @@ public boolean hasAuthenticationToken(final HttpServletRequest request) { return false; } - @Override - public void removeAuthorisationEntries(final Map headers) { - - } - @Override public Map getAuthHeaders(final UserIdentity userIdentity) { return Collections.emptyMap(); diff --git a/stroom-app/src/test/java/stroom/receive/TestReceiveDataServlet.java b/stroom-app/src/test/java/stroom/receive/TestReceiveDataServlet.java index 088d948ec33..dca572050de 100644 --- a/stroom-app/src/test/java/stroom/receive/TestReceiveDataServlet.java +++ b/stroom-app/src/test/java/stroom/receive/TestReceiveDataServlet.java @@ -288,7 +288,7 @@ void testOKCompressionZeroContent() throws IOException, ServletException { request.addHeader("feed", "TEST-FEED"); request.addHeader("periodStartTime", DateUtil.createNormalDateTimeString()); request.addHeader("periodEndTime", DateUtil.createNormalDateTimeString()); - request.addHeader(StandardHeaderArguments.CONTENT_LENGTH, "0"); + request.addHeader(StandardHeaderArguments.CONTENT_LENGTH.get(), "0"); request.addHeader("compression", "GZIP"); request.setInputStream("".getBytes()); diff --git a/stroom-app/src/test/java/stroom/search/AbstractInteractiveSearchTest.java b/stroom-app/src/test/java/stroom/search/AbstractInteractiveSearchTest.java index 229cfcc7c73..32ec4ffbaa7 100644 --- a/stroom-app/src/test/java/stroom/search/AbstractInteractiveSearchTest.java +++ b/stroom-app/src/test/java/stroom/search/AbstractInteractiveSearchTest.java @@ -524,7 +524,7 @@ private TableSettings createTableSettings(final boolean extractValues) { final Column statusColumn = Column.builder() .name("Status") - .expression(ParamSubstituteUtil.makeParam(AnnotationFields.STATUS)) + .expression(ParamSubstituteUtil.makeParam(AnnotationFields.STATUS_FIELD.getFldName())) .build(); final DocRef resultPipeline = commonIndexingTestHelper.getSearchResultPipeline(); diff --git a/stroom-app/src/test/java/stroom/search/TestInteractiveSearch2.java b/stroom-app/src/test/java/stroom/search/TestInteractiveSearch2.java index d17d7b171ed..d2bf9df1fa0 100644 --- a/stroom-app/src/test/java/stroom/search/TestInteractiveSearch2.java +++ b/stroom-app/src/test/java/stroom/search/TestInteractiveSearch2.java @@ -460,7 +460,7 @@ private TableSettings createTableSettings(final boolean extractValues) { final Column statusColumn = Column.builder() .name("Status") - .expression(ParamSubstituteUtil.makeParam(AnnotationFields.STATUS)) + .expression(ParamSubstituteUtil.makeParam(AnnotationFields.STATUS_FIELD.getFldName())) .build(); final DocRef resultPipeline = commonIndexingTestHelper.getSearchResultPipeline(); diff --git a/stroom-app/src/test/java/stroom/state/TestSessionDao.java b/stroom-app/src/test/java/stroom/state/impl/dao/TestSessionDao.java similarity index 91% rename from stroom-app/src/test/java/stroom/state/TestSessionDao.java rename to stroom-app/src/test/java/stroom/state/impl/dao/TestSessionDao.java index 9ac415ca988..9ac6f047b64 100644 --- a/stroom-app/src/test/java/stroom/state/TestSessionDao.java +++ b/stroom-app/src/test/java/stroom/state/impl/dao/TestSessionDao.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,10 +12,9 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package stroom.state; +package stroom.state.impl.dao; import stroom.entity.shared.ExpressionCriteria; import stroom.query.api.v2.ExpressionOperator; @@ -23,11 +22,8 @@ import stroom.query.language.functions.FieldIndex; import stroom.query.language.functions.ValDate; import stroom.state.impl.ScyllaDbUtil; -import stroom.state.impl.dao.Session; -import stroom.state.impl.dao.SessionDao; -import stroom.state.impl.dao.SessionFields; -import stroom.state.impl.dao.TemporalStateRequest; import stroom.test.AbstractCoreIntegrationTest; +import stroom.util.shared.string.CIKeys; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -85,10 +81,10 @@ void testDao() { final AtomicInteger count = new AtomicInteger(); final FieldIndex fieldIndex = new FieldIndex(); - fieldIndex.create(SessionFields.KEY); - fieldIndex.create(SessionFields.START); - fieldIndex.create(SessionFields.END); - fieldIndex.create(SessionFields.TERMINAL); + fieldIndex.create(CIKeys.KEY); + fieldIndex.create(CIKeys.START); + fieldIndex.create(CIKeys.END); + fieldIndex.create(CIKeys.TERMINAL); final ValDate minTime = ValDate.create(min); final ValDate maxTime = ValDate.create(max); diff --git a/stroom-app/src/test/java/stroom/state/TestStateDao.java b/stroom-app/src/test/java/stroom/state/impl/dao/TestStateDao.java similarity index 92% rename from stroom-app/src/test/java/stroom/state/TestStateDao.java rename to stroom-app/src/test/java/stroom/state/impl/dao/TestStateDao.java index 1a69895de77..50d7824fa22 100644 --- a/stroom-app/src/test/java/stroom/state/TestStateDao.java +++ b/stroom-app/src/test/java/stroom/state/impl/dao/TestStateDao.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,16 +12,12 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package stroom.state; +package stroom.state.impl.dao; import stroom.pipeline.refdata.store.StringValue; import stroom.state.impl.ScyllaDbUtil; -import stroom.state.impl.dao.TemporalState; -import stroom.state.impl.dao.TemporalStateDao; -import stroom.state.impl.dao.TemporalStateRequest; import stroom.test.AbstractCoreIntegrationTest; import org.junit.jupiter.api.Test; diff --git a/stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3ConfigStoreImpl.java b/stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3ConfigStoreImpl.java index ec73a74e9a3..1b0caf269e0 100644 --- a/stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3ConfigStoreImpl.java +++ b/stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3ConfigStoreImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Crown Copyright + * Copyright 2019-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -214,8 +214,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3Manager.java b/stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3Manager.java index cc48b13d6c2..0d3f5912450 100644 --- a/stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3Manager.java +++ b/stroom-aws/stroom-aws-s3-impl/src/main/java/stroom/aws/s3/impl/S3Manager.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.aws.s3.impl; import stroom.aws.s3.shared.AwsAssumeRole; @@ -676,7 +692,7 @@ private PutObjectRequest createPutObjectRequest(final String bucketName, final String key, final Meta meta, final AttributeMap attributeMap) { - final Map metadata = attributeMap + final Map metadata = attributeMap.asUnmodifiableStringKeyedMap() .entrySet() .stream() .collect(Collectors.toMap(e -> createS3Name(e.getKey()), Entry::getValue)); diff --git a/stroom-cache/stroom-cache-impl/src/main/java/stroom/cache/impl/CacheManagerImpl.java b/stroom-cache/stroom-cache-impl/src/main/java/stroom/cache/impl/CacheManagerImpl.java index 246742e210a..ef07df3f80e 100644 --- a/stroom-cache/stroom-cache-impl/src/main/java/stroom/cache/impl/CacheManagerImpl.java +++ b/stroom-cache/stroom-cache-impl/src/main/java/stroom/cache/impl/CacheManagerImpl.java @@ -147,62 +147,81 @@ public SystemInfoResult getSystemInfo(final Map params) { final String cacheName = params.get(PARAM_NAME_CACHE_NAME); if (cacheName != null) { - final StroomCache cache = caches.get(cacheName); + return getSystemInfoByCacheName(cacheName, limit); + } else { + return getSystemInfoCacheList(limit); + } + } + + private SystemInfoResult getSystemInfoCacheList(final Integer limit) { + final List cacheNames = caches.keySet() + .stream() + .sorted() + .limit(limit) + .collect(Collectors.toList()); + + return SystemInfoResult.builder(this) + .description("List of cache names") + .addDetail("cacheNames", cacheNames) + .addDetail("cacheCount", caches.size()) + .build(); + } - if (cache != null) { - final Set keySet = cache.keySet(); + private SystemInfoResult getSystemInfoByCacheName(final String cacheName, final Integer limit) { + final StroomCache cache = caches.get(cacheName); - Stream stream = keySet - .stream() - .limit(limit); + if (cache != null) { + final Set keySet = cache.keySet(); + + Stream stream = keySet + .stream() + .limit(limit); - final List keyList; - if (!keySet.isEmpty()) { - final Object aKey = keySet.iterator().next(); + final List keyList; + if (!keySet.isEmpty()) { + final Object aKey = keySet.iterator().next(); - if (aKey instanceof Comparable) { - stream = stream - .sorted(); - } - keyList = stream - .map(key -> { + if (aKey instanceof Comparable) { + stream = stream + .sorted(); + } + keyList = stream + .map(key -> { + if (key != null) { + Object result = null; try { // Try and serialise it - JsonUtil.writeValueAsString(key); + final String json = JsonUtil.writeValueAsString(key); + if (json != null) { + result = key; + } } catch (Exception e) { - return "Unable to serialise Key as JSON, dumping as string: " - + key.toString().substring(0, 1_000); + // swallow } - return key; - }) - .collect(Collectors.toList()); - - } else { - keyList = Collections.emptyList(); - } + if (result == null) { + final String str = key.toString(); + result = "Unable to serialise Key as JSON, dumping as string: " + + key.toString().substring(0, Math.min(str.length(), 1_000)); + } + return result; + } else { + return null; + } + }) + .collect(Collectors.toList()); - return SystemInfoResult.builder(this) - .description("List of cache keys") - .addDetail("cacheName", cacheName) - .addDetail("keys", keyList) - .addDetail("keyCount", keySet.size()) - .build(); } else { - throw new RuntimeException(LogUtil.message("Unknown cache name {}", cacheName)); + keyList = Collections.emptyList(); } - } else { - final List cacheNames = caches.keySet() - .stream() - .sorted() - .limit(limit) - .collect(Collectors.toList()); - return SystemInfoResult.builder(this) - .description("List of cache names") - .addDetail("cacheNames", cacheNames) - .addDetail("cacheCount", caches.size()) + .description("List of cache keys") + .addDetail("cacheName", cacheName) + .addDetail("keys", keyList) + .addDetail("keyCount", keySet.size()) .build(); + } else { + throw new RuntimeException(LogUtil.message("Unknown cache name {}", cacheName)); } } diff --git a/stroom-core-client-widget/src/main/java/stroom/widget/menu/client/presenter/Menu.java b/stroom-core-client-widget/src/main/java/stroom/widget/menu/client/presenter/Menu.java index f1332f917d8..a3ef005a933 100644 --- a/stroom-core-client-widget/src/main/java/stroom/widget/menu/client/presenter/Menu.java +++ b/stroom-core-client-widget/src/main/java/stroom/widget/menu/client/presenter/Menu.java @@ -1,5 +1,6 @@ package stroom.widget.menu.client.presenter; +import stroom.util.shared.GwtNullSafe; import stroom.widget.popup.client.event.ShowPopupEvent; import stroom.widget.popup.client.presenter.PopupType; @@ -42,7 +43,7 @@ private void show(final ShowMenuEvent event) { .popupType(PopupType.POPUP) .popupPosition(event.getPopupPosition()) .addAutoHidePartner(event.getAutoHidePartners()) - .onShow(e -> menuPresenter.focus()) + .onShow(e -> GwtNullSafe.consume(menuPresenter, MenuPresenter::focus)) .onHide(e -> { if (event.getHideHandler() != null) { event.getHideHandler().onHide(e); diff --git a/stroom-core-client/src/main/java/stroom/dashboard/client/HasSelection.java b/stroom-core-client/src/main/java/stroom/dashboard/client/HasSelection.java index 5e81d704b5c..bbd05307e4b 100644 --- a/stroom-core-client/src/main/java/stroom/dashboard/client/HasSelection.java +++ b/stroom-core-client/src/main/java/stroom/dashboard/client/HasSelection.java @@ -1,6 +1,23 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.dashboard.client; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; import java.util.List; import java.util.Map; @@ -9,5 +26,5 @@ public interface HasSelection { List getFields(); - List> getSelection(); + List> getSelection(); } diff --git a/stroom-core-client/src/main/java/stroom/dashboard/client/query/QueryPresenter.java b/stroom-core-client/src/main/java/stroom/dashboard/client/query/QueryPresenter.java index 7bdb4f12c43..5c3b5ec5202 100644 --- a/stroom-core-client/src/main/java/stroom/dashboard/client/query/QueryPresenter.java +++ b/stroom-core-client/src/main/java/stroom/dashboard/client/query/QueryPresenter.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -74,6 +74,7 @@ import stroom.ui.config.client.UiConfigCache; import stroom.util.shared.EqualsBuilder; import stroom.util.shared.ModelStringUtil; +import stroom.util.shared.string.CIKey; import stroom.widget.button.client.ButtonView; import stroom.widget.menu.client.presenter.IconMenuItem; import stroom.widget.menu.client.presenter.Item; @@ -300,8 +301,9 @@ public void setComponents(final Components components) { if (initialised) { final Component component = event.getComponent(); if (component instanceof HasSelection) { + //noinspection PatternVariableCanBeUsed // GWT final HasSelection hasSelection = (HasSelection) component; - final List> selection = hasSelection.getSelection(); + final List> selection = hasSelection.getSelection(); final List selectionHandlers = getQuerySettings().getSelectionHandlers(); if (selectionHandlers != null) { final List matchingHandlers = selectionHandlers @@ -311,13 +313,13 @@ public void setComponents(final Components components) { selectionHandler.getComponentId().equals(component.getId())) .collect(Collectors.toList()); - if (matchingHandlers.size() > 0) { + if (!matchingHandlers.isEmpty()) { final Function decorator = (in) -> { final ExpressionOperator.Builder innerBuilder = ExpressionOperator .builder(); boolean added = false; for (final ComponentSelectionHandler selectionHandler : matchingHandlers) { - for (final Map params : selection) { + for (final Map params : selection) { ExpressionOperator ex = selectionHandler.getExpression(); ex = ExpressionUtil.replaceExpressionParameters(ex, params); innerBuilder.addOperator(ex); diff --git a/stroom-core-client/src/main/java/stroom/dashboard/client/table/AnnotationManager.java b/stroom-core-client/src/main/java/stroom/dashboard/client/table/AnnotationManager.java index 3213379ff23..064f26e5029 100644 --- a/stroom-core-client/src/main/java/stroom/dashboard/client/table/AnnotationManager.java +++ b/stroom-core-client/src/main/java/stroom/dashboard/client/table/AnnotationManager.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,8 @@ import stroom.query.api.v2.Column; import stroom.query.client.presenter.TableRow; import stroom.svg.shared.SvgImage; +import stroom.util.shared.GwtNullSafe; +import stroom.util.shared.string.CIKey; import stroom.widget.menu.client.presenter.IconMenuItem; import stroom.widget.menu.client.presenter.Item; import stroom.widget.menu.client.presenter.ShowMenuEvent; @@ -144,12 +146,14 @@ public List getEventIdList(final TableComponentSettings tableComponentS } private String getFieldId(final TableComponentSettings tableComponentSettings, final String fieldName) { - for (final Column column : tableComponentSettings.getColumns()) { - if (column.getName().equalsIgnoreCase(fieldName)) { - return column.getId(); - } - } - return null; + final CIKey fieldNameKey = CIKey.of(fieldName); + return tableComponentSettings.getColumns() + .stream() + .filter(column -> + Objects.equals(column.getNameAsCIKey(), fieldNameKey)) + .findAny() + .map(Column::getId) + .orElse(null); } public List getAnnotationIdList(final TableComponentSettings tableComponentSettings, @@ -166,7 +170,7 @@ public List getValues(final TableComponentSettings tableComponentSetting final List selectedItems, final String fieldName) { final List values = new ArrayList<>(); - if (selectedItems != null && selectedItems.size() > 0) { + if (!GwtNullSafe.isEmptyCollection(selectedItems)) { final String fieldId = getFieldId(tableComponentSettings, fieldName); if (fieldId != null) { for (final TableRow row : selectedItems) { @@ -181,7 +185,7 @@ public List getValues(final TableComponentSettings tableComponentSetting public String getValue(final TableComponentSettings tableComponentSettings, final List selectedItems, final String fieldName) { - if (selectedItems != null && selectedItems.size() > 0) { + if (!GwtNullSafe.isEmptyCollection(selectedItems)) { final String fieldId = getFieldId(tableComponentSettings, fieldName); if (fieldId != null) { for (final TableRow row : selectedItems) { diff --git a/stroom-core-client/src/main/java/stroom/dashboard/client/table/TablePresenter.java b/stroom-core-client/src/main/java/stroom/dashboard/client/table/TablePresenter.java index 238e77e7d96..26929a46a5f 100644 --- a/stroom-core-client/src/main/java/stroom/dashboard/client/table/TablePresenter.java +++ b/stroom-core-client/src/main/java/stroom/dashboard/client/table/TablePresenter.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.dashboard.client.table; @@ -84,6 +83,7 @@ import stroom.ui.config.shared.UserPreferences; import stroom.util.shared.Expander; import stroom.util.shared.Version; +import stroom.util.shared.string.CIKey; import stroom.widget.button.client.ButtonView; import stroom.widget.popup.client.event.ShowPopupEvent; import stroom.widget.popup.client.presenter.PopupType; @@ -583,7 +583,7 @@ private List processData(final List columns, final List v rowStyle.trustedColor(row.getTextColor()); } - final Map cellsMap = new HashMap<>(); + final Map fieldIdToCellMap = new HashMap<>(); for (int i = 0; i < columns.size() && i < row.getValues().size(); i++) { final Column column = columns.get(i); final String value = row.getValues().get(i) != null @@ -607,8 +607,8 @@ private List processData(final List columns, final List v final String style = stylesBuilder.toSafeStyles().asString(); final TableRow.Cell cell = new TableRow.Cell(value, style); - cellsMap.put(column.getName(), cell); - cellsMap.put(column.getId(), cell); + fieldIdToCellMap.put(column.getName(), cell); + fieldIdToCellMap.put(column.getId(), cell); } // Create an expander for the row. @@ -620,7 +620,7 @@ private List processData(final List columns, final List v expander = new Expander(row.getDepth(), false, true); } - processed.add(new TableRow(expander, row.getGroupKey(), cellsMap)); + processed.add(new TableRow(expander, row.getGroupKey(), fieldIdToCellMap)); } // Set the expander column width. @@ -770,7 +770,8 @@ private void updateFields() { private void ensureSpecialFields(final String... indexFieldNames) { // Remove all special fields as we will re-add them. - getTableSettings().getColumns().removeIf(Column::isSpecial); + getTableSettings().getColumns() + .removeIf(Column::isSpecial); final Optional maxGroup = getTableSettings() .getColumns() @@ -778,6 +779,7 @@ private void ensureSpecialFields(final String... indexFieldNames) { .map(Column::getGroup) .filter(Objects::nonNull) .max(Integer::compareTo); + if (getTableSettings().showDetail() || maxGroup.isEmpty()) { final List requiredSpecialColumns = new ArrayList<>(); for (final String indexFieldName : indexFieldNames) { @@ -787,7 +789,7 @@ private void ensureSpecialFields(final String... indexFieldNames) { // If the fields we want to make special do exist in the current data source then // add them. - if (requiredSpecialColumns.size() > 0) { + if (!requiredSpecialColumns.isEmpty()) { // Prior to the introduction of the special field concept, special fields were // treated as invisible fields. For this reason we need to remove old invisible // fields if we haven't yet turned them into special fields. @@ -993,13 +995,13 @@ public List getFields() { } @Override - public List> getSelection() { - final List> list = new ArrayList<>(); + public List> getSelection() { + final List> list = new ArrayList<>(); final List columns = getTableSettings().getColumns(); for (final TableRow tableRow : getSelectedRows()) { - final Map map = new HashMap<>(); + final Map map = new HashMap<>(); for (final Column column : columns) { - map.put(column.getName(), tableRow.getText(column.getId())); + map.put(column.getNameAsCIKey(), tableRow.getText(column.getId())); } list.add(map); } diff --git a/stroom-core-client/src/main/java/stroom/dashboard/client/vis/MessageSupport.java b/stroom-core-client/src/main/java/stroom/dashboard/client/vis/MessageSupport.java index 7fb8ed558ac..bd13ec0aecc 100644 --- a/stroom-core-client/src/main/java/stroom/dashboard/client/vis/MessageSupport.java +++ b/stroom-core-client/src/main/java/stroom/dashboard/client/vis/MessageSupport.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import stroom.task.client.HasTaskMonitorFactory; import stroom.task.client.TaskMonitorFactory; import stroom.util.client.JSONUtil; +import stroom.util.shared.string.CIKey; import com.google.gwt.core.client.Callback; import com.google.gwt.core.client.GWT; @@ -37,6 +38,7 @@ import com.gwtplatform.mvp.client.HasUiHandlers; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -90,19 +92,21 @@ public void postMessage(final JSONObject json) { PostMessage.get().postMessage(frame, json, frameId); } - private Map toMap(final JSONObject obj) { - final Map map = new HashMap<>(); + private Map toMap(final JSONObject obj) { if (obj != null) { + final Map map = new HashMap<>(); for (final String key : obj.keySet()) { final JSONValue v = obj.get(key); if (v.isString() != null) { - map.put(key, v.isString().stringValue()); + map.put(CIKey.of(key), v.isString().stringValue()); } else { - map.put(key, v.toString()); + map.put(CIKey.of(key), v.toString()); } } + return map; + } else { + return Collections.emptyMap(); } - return map; } @Override @@ -121,7 +125,7 @@ public void receiveMessage(final MessageEvent event, final JSONObject message) { GWT.log("Received selection from vis: " + selection.toString()); if (uiHandlers != null) { - final List> list = new ArrayList<>(); + final List> list = new ArrayList<>(); final JSONArray array = selection.isArray(); if (array != null) { for (int i = 0; i < array.size(); i++) { diff --git a/stroom-core-client/src/main/java/stroom/dashboard/client/vis/SelectionUiHandlers.java b/stroom-core-client/src/main/java/stroom/dashboard/client/vis/SelectionUiHandlers.java index 1d6800b0265..cc437b115c1 100644 --- a/stroom-core-client/src/main/java/stroom/dashboard/client/vis/SelectionUiHandlers.java +++ b/stroom-core-client/src/main/java/stroom/dashboard/client/vis/SelectionUiHandlers.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package stroom.dashboard.client.vis; +import stroom.util.shared.string.CIKey; + import com.gwtplatform.mvp.client.UiHandlers; import java.util.List; @@ -23,5 +25,5 @@ public interface SelectionUiHandlers extends UiHandlers { - void onSelection(List> values); + void onSelection(List> values); } diff --git a/stroom-core-client/src/main/java/stroom/dashboard/client/vis/VisPresenter.java b/stroom-core-client/src/main/java/stroom/dashboard/client/vis/VisPresenter.java index 8eca2ac6d8f..736d3f245e5 100644 --- a/stroom-core-client/src/main/java/stroom/dashboard/client/vis/VisPresenter.java +++ b/stroom-core-client/src/main/java/stroom/dashboard/client/vis/VisPresenter.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,6 +48,7 @@ import stroom.ui.config.shared.Theme; import stroom.util.client.JSONUtil; import stroom.util.shared.EqualsUtil; +import stroom.util.shared.string.CIKey; import stroom.visualisation.client.presenter.VisFunction; import stroom.visualisation.client.presenter.VisFunction.LoadStatus; import stroom.visualisation.client.presenter.VisFunction.StatusHandler; @@ -122,7 +123,7 @@ public class VisPresenter private TablePresenter linkedTablePresenter; private final Timer timer; - private List> currentSelection; + private List> currentSelection; private boolean pause; @Inject @@ -152,7 +153,7 @@ public void run() { } @Override - public void onSelection(final List> selection) { + public void onSelection(final List> selection) { if (!Objects.equals(currentSelection, selection)) { currentSelection = selection; timer.schedule(250); @@ -731,7 +732,7 @@ public List getFields() { } @Override - public List> getSelection() { + public List> getSelection() { return currentSelection; } diff --git a/stroom-core-client/src/main/java/stroom/data/client/presenter/AbstractMetaListPresenter.java b/stroom-core-client/src/main/java/stroom/data/client/presenter/AbstractMetaListPresenter.java index e42729aa61f..14e03774e6a 100644 --- a/stroom-core-client/src/main/java/stroom/data/client/presenter/AbstractMetaListPresenter.java +++ b/stroom-core-client/src/main/java/stroom/data/client/presenter/AbstractMetaListPresenter.java @@ -70,7 +70,6 @@ import com.google.gwt.cell.client.TextCell; import com.google.gwt.core.shared.GWT; -import com.google.gwt.safehtml.shared.SafeHtml; import com.google.gwt.user.cellview.client.Column; import com.google.gwt.user.cellview.client.Header; import com.google.gwt.view.client.Range; @@ -350,7 +349,7 @@ void addCreatedColumn() { } void addFeedColumn() { - dataGrid.addResizableColumn( + dataGrid.addAutoResizableColumn( DataGridUtil.docRefColumnBuilder((MetaRow metaRow) -> Optional.ofNullable(metaRow) .map(this::getFeed) @@ -409,7 +408,7 @@ void addPipelineColumn() { .withSorting(MetaFields.PIPELINE_NAME) .build(), "Pipeline", - ColumnSizeConstants.BIG_COL); + 350); } protected MultiSelectionModel getSelectionModel() { @@ -470,46 +469,46 @@ void addRightAlignedAttributeColumn(final String name, size); } - void addColouredSizeAttributeColumn(final String name, - final QueryField attribute, - final Function formatter, - final int size) { - - final Function extractor = metaRow -> { - final String value = metaRow.getAttributeValue(attribute.getFldName()); - if (value == null) { - return null; - } else { - return formatter.apply(value); - } - }; - - final Function colourFunc = val -> { - if (val == null) { - return "black"; - } else if (val.endsWith("B")) { - return "blue"; - } else if (val.endsWith("K")) { - return "green"; - } else if (val.endsWith("M")) { - return "#FF7F00"; - } else if (val.endsWith("G")) { - return "red"; - } else { - return "red"; - } - }; - - final Column column = DataGridUtil.htmlColumnBuilder( - DataGridUtil.colouredCellExtractor(extractor, colourFunc)) - .rightAligned() - .build(); - - dataGrid.addResizableColumn( - column, - DataGridUtil.createRightAlignedHeader(name), - size); - } +// void addColouredSizeAttributeColumn(final String name, +// final QueryField attribute, +// final Function formatter, +// final int size) { +// +// final Function extractor = metaRow -> { +// final String value = metaRow.getAttributeValue(attribute.getFldName()); +// if (value == null) { +// return null; +// } else { +// return formatter.apply(value); +// } +// }; +// +// final Function colourFunc = val -> { +// if (val == null) { +// return "black"; +// } else if (val.endsWith("B")) { +// return "blue"; +// } else if (val.endsWith("K")) { +// return "green"; +// } else if (val.endsWith("M")) { +// return "#FF7F00"; +// } else if (val.endsWith("G")) { +// return "red"; +// } else { +// return "red"; +// } +// }; +// +// final Column column = DataGridUtil.htmlColumnBuilder( +// DataGridUtil.colouredCellExtractor(extractor, colourFunc)) +// .rightAligned() +// .build(); +// +// dataGrid.addResizableColumn( +// column, +// DataGridUtil.createRightAlignedHeader(name), +// size); +// } void addEndColumn() { dataGrid.addEndColumn(new EndColumn<>()); diff --git a/stroom-core-client/src/main/java/stroom/data/client/presenter/ProcessorTaskListPresenter.java b/stroom-core-client/src/main/java/stroom/data/client/presenter/ProcessorTaskListPresenter.java index 7c17374b8ed..619682cb7fb 100644 --- a/stroom-core-client/src/main/java/stroom/data/client/presenter/ProcessorTaskListPresenter.java +++ b/stroom-core-client/src/main/java/stroom/data/client/presenter/ProcessorTaskListPresenter.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -167,7 +167,9 @@ public String getValue(final ProcessorTask row) { }, "Node", ColumnSizeConstants.MEDIUM_COL); dataGrid .addResizableColumn(new OrderByColumn( - new DocRefCell(getEventBus(), true), ProcessorTaskFields.FIELD_FEED, true) { + new DocRefCell(getEventBus(), true), + ProcessorTaskFields.FIELD_FEED, + true) { @Override public DocRef getValue(final ProcessorTask row) { if (row.getFeedName() != null) { @@ -219,6 +221,7 @@ public String getValue(final ProcessorTask row) { dataGrid.addColumnSortHandler(event -> { if (event.getColumn() instanceof OrderByColumn) { + //noinspection PatternVariableCanBeUsed // GWT final OrderByColumn orderByColumn = (OrderByColumn) event.getColumn(); criteria.setSort(orderByColumn.getField(), !event.isSortAscending(), orderByColumn.isIgnoreCase()); refresh(); diff --git a/stroom-core-client/src/main/java/stroom/query/client/presenter/AnnotationFields.java b/stroom-core-client/src/main/java/stroom/query/client/presenter/AnnotationFields.java index 8705abf5f53..758d52724d5 100644 --- a/stroom-core-client/src/main/java/stroom/query/client/presenter/AnnotationFields.java +++ b/stroom-core-client/src/main/java/stroom/query/client/presenter/AnnotationFields.java @@ -1,6 +1,7 @@ package stroom.query.client.presenter; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; import java.util.Arrays; import java.util.List; @@ -26,19 +27,19 @@ public interface AnnotationFields { String COMMENT = ANNOTATION_FIELD_PREFIX + "Comment"; String HISTORY = ANNOTATION_FIELD_PREFIX + "History"; - QueryField ID_FIELD = QueryField.createId(ID); + QueryField ID_FIELD = QueryField.createId(CIKey.ofStaticKey(ID), true); // AbstractField STREAM_ID_FIELD = QueryField.createId(IndexConstants.STREAM_ID); // AbstractField EVENT_ID_FIELD = QueryField.createId(IndexConstants.EVENT_ID); - QueryField CREATED_ON_FIELD = QueryField.createDate(CREATED_ON); - QueryField CREATED_BY_FIELD = QueryField.createText(CREATED_BY); - QueryField UPDATED_ON_FIELD = QueryField.createDate(UPDATED_ON); - QueryField UPDATED_BY_FIELD = QueryField.createText(UPDATED_BY); - QueryField TITLE_FIELD = QueryField.createText(TITLE); - QueryField SUBJECT_FIELD = QueryField.createText(SUBJECT); - QueryField STATUS_FIELD = QueryField.createText(STATUS); - QueryField ASSIGNED_TO_FIELD = QueryField.createText(ASSIGNED_TO); - QueryField COMMENT_FIELD = QueryField.createText(COMMENT); - QueryField HISTORY_FIELD = QueryField.createText(HISTORY); + QueryField CREATED_ON_FIELD = QueryField.createDate(CIKey.ofStaticKey(CREATED_ON), true); + QueryField CREATED_BY_FIELD = QueryField.createText(CIKey.ofStaticKey(CREATED_BY), true); + QueryField UPDATED_ON_FIELD = QueryField.createDate(CIKey.ofStaticKey(UPDATED_ON), true); + QueryField UPDATED_BY_FIELD = QueryField.createText(CIKey.ofStaticKey(UPDATED_BY), true); + QueryField TITLE_FIELD = QueryField.createText(CIKey.ofStaticKey(TITLE), true); + QueryField SUBJECT_FIELD = QueryField.createText(CIKey.ofStaticKey(SUBJECT), true); + QueryField STATUS_FIELD = QueryField.createText(CIKey.ofStaticKey(STATUS), true); + QueryField ASSIGNED_TO_FIELD = QueryField.createText(CIKey.ofStaticKey(ASSIGNED_TO), true); + QueryField COMMENT_FIELD = QueryField.createText(CIKey.ofStaticKey(COMMENT), true); + QueryField HISTORY_FIELD = QueryField.createText(CIKey.ofStaticKey(HISTORY), true); List FIELDS = Arrays.asList( ID_FIELD, @@ -54,6 +55,7 @@ public interface AnnotationFields { ASSIGNED_TO_FIELD, COMMENT_FIELD, HISTORY_FIELD); + Map FIELD_MAP = FIELDS.stream() .collect(Collectors.toMap(QueryField::getFldName, Function.identity())); } diff --git a/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicColumnSelectionListModel.java b/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicColumnSelectionListModel.java index 2510006acce..f6cbab03020 100644 --- a/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicColumnSelectionListModel.java +++ b/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicColumnSelectionListModel.java @@ -67,7 +67,7 @@ public void onRangeChange(final ColumnSelectionItem parent, final Consumer> consumer) { final String parentPath = getParentPath(parent); if (dataSourceRef != null) { - final StringMatch stringMatch = StringMatch.contains(filter); + final StringMatch stringMatch = StringMatch.containsIgnoreCase(filter); final FindFieldCriteria findFieldInfoCriteria = new FindFieldCriteria( pageRequest, null, diff --git a/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicFieldSelectionListModel.java b/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicFieldSelectionListModel.java index e71984548a0..031386bbf44 100644 --- a/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicFieldSelectionListModel.java +++ b/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicFieldSelectionListModel.java @@ -46,7 +46,7 @@ public void onRangeChange(final FieldInfoSelectionItem parent, final PageRequest pageRequest, final Consumer> consumer) { if (dataSourceRef != null) { - final StringMatch stringMatch = StringMatch.contains(filter); + final StringMatch stringMatch = StringMatch.containsIgnoreCase(filter); final FindFieldCriteria findFieldInfoCriteria = new FindFieldCriteria( pageRequest, null, diff --git a/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicQueryHelpSelectionListModel.java b/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicQueryHelpSelectionListModel.java index 3373a45a20d..d53b4779d75 100644 --- a/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicQueryHelpSelectionListModel.java +++ b/stroom-core-client/src/main/java/stroom/query/client/presenter/DynamicQueryHelpSelectionListModel.java @@ -88,7 +88,7 @@ public void onRangeChange(final QueryHelpSelectionItem parent, parentId = ""; } - final StringMatch stringMatch = StringMatch.contains(filter); + final StringMatch stringMatch = StringMatch.containsIgnoreCase(filter); final CriteriaFieldSort sort = new CriteriaFieldSort( FindFieldCriteria.SORT_BY_NAME, false, diff --git a/stroom-core-client/src/main/java/stroom/query/client/presenter/QueryResultTablePresenter.java b/stroom-core-client/src/main/java/stroom/query/client/presenter/QueryResultTablePresenter.java index a9d05befe00..fe898357b18 100644 --- a/stroom-core-client/src/main/java/stroom/query/client/presenter/QueryResultTablePresenter.java +++ b/stroom-core-client/src/main/java/stroom/query/client/presenter/QueryResultTablePresenter.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -498,7 +498,7 @@ private List processData(final List columns, final List v rowStyle.trustedColor(row.getTextColor()); } - final Map cellsMap = new HashMap<>(); + final Map fieldIdToCellMap = new HashMap<>(); for (int i = 0; i < columns.size() && i < row.getValues().size(); i++) { final Column column = columns.get(i); final String value = row.getValues().get(i) != null @@ -522,8 +522,8 @@ private List processData(final List columns, final List v final String style = stylesBuilder.toSafeStyles().asString(); final TableRow.Cell cell = new TableRow.Cell(value, style); - cellsMap.put(column.getName(), cell); - cellsMap.put(column.getId(), cell); + fieldIdToCellMap.put(column.getName(), cell); + fieldIdToCellMap.put(column.getId(), cell); } // Create an expander for the row. @@ -535,7 +535,7 @@ private List processData(final List columns, final List v expander = new Expander(row.getDepth(), false, true); } - processed.add(new TableRow(expander, row.getGroupKey(), cellsMap)); + processed.add(new TableRow(expander, row.getGroupKey(), fieldIdToCellMap)); } // Set the expander column width. diff --git a/stroom-core-client/src/main/java/stroom/query/client/presenter/TableRow.java b/stroom-core-client/src/main/java/stroom/query/client/presenter/TableRow.java index 857cfc1c5b5..617d48a1f09 100644 --- a/stroom-core-client/src/main/java/stroom/query/client/presenter/TableRow.java +++ b/stroom-core-client/src/main/java/stroom/query/client/presenter/TableRow.java @@ -1,7 +1,24 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.client.presenter; import stroom.hyperlink.client.Hyperlink; import stroom.util.shared.Expander; +import stroom.util.shared.GwtNullSafe; import stroom.widget.util.client.SafeHtmlUtil; import com.google.gwt.safehtml.shared.SafeHtml; @@ -17,14 +34,14 @@ public class TableRow { private final Expander expander; private final String groupKey; - private final Map cells; + private final Map columnIdToCellMap; public TableRow(final Expander expander, final String groupKey, - final Map cells) { + final Map columnIdToCellMap) { this.expander = expander; this.groupKey = groupKey; - this.cells = cells; + this.columnIdToCellMap = columnIdToCellMap; } public Expander getExpander() { @@ -35,9 +52,9 @@ public String getGroupKey() { return groupKey; } - public SafeHtml getValue(final String fieldId) { + public SafeHtml getValue(final String columnId) { // Turn the raw value into html on demand - final Cell cell = cells.get(fieldId); + final Cell cell = columnIdToCellMap.get(columnId); if (cell != null) { return decorateValue(cell); } else { @@ -65,8 +82,8 @@ private SafeHtml decorateValue(final Cell cell) { return safeHtmlBuilder.toSafeHtml(); } - public String getText(final String fieldId) { - final Cell cell = cells.get(fieldId); + public String getText(final String columnId) { + final Cell cell = columnIdToCellMap.get(columnId); if (cell != null) { final String rawValue = cell.getRawValue(); if (rawValue != null) { @@ -90,14 +107,15 @@ public String getText(final String fieldId) { private void appendValue(final String value, final SafeHtmlBuilder sb) { final List parts = getParts(value); - if (parts.size() == 0) { + if (parts.isEmpty()) { appendText(value, sb); } else { parts.forEach(p -> { if (p instanceof Hyperlink) { + //noinspection PatternVariableCanBeUsed // GWT final Hyperlink hyperlink = (Hyperlink) p; - if (!hyperlink.getText().trim().isEmpty()) { + if (GwtNullSafe.isNonBlankString(hyperlink.getText())) { sb.appendHtmlConstant(""); appendText(hyperlink.getText(), sb); sb.appendHtmlConstant(""); @@ -110,7 +128,7 @@ private void appendValue(final String value, final SafeHtmlBuilder sb) { } private void appendText(final String text, final SafeHtmlBuilder sb) { - if (text == null || text.trim().length() == 0) { + if (GwtNullSafe.isBlankString(text)) { sb.append(SafeHtmlUtil.NBSP); } else { sb.appendEscaped(text); @@ -128,6 +146,7 @@ private List getParts(final String value) { if (c == '[') { final Hyperlink hyperlink = Hyperlink.create(value, i); if (hyperlink != null) { + //noinspection SizeReplaceableByIsEmpty // GWT if (sb.length() > 0) { parts.add(sb.toString()); sb.setLength(0); @@ -143,6 +162,7 @@ private List getParts(final String value) { } } + //noinspection SizeReplaceableByIsEmpty // GWT if (sb.length() > 0) { parts.add(sb.toString()); } @@ -155,7 +175,7 @@ public String toString() { return "TableRow{" + "expander=" + expander + ", groupKey='" + groupKey + '\'' + - ", cells=" + cells + + ", cells=" + columnIdToCellMap + '}'; } @@ -176,6 +196,10 @@ public int hashCode() { return Objects.hash(groupKey); } + + // -------------------------------------------------------------------------------- + + public static class Cell { private final String rawValue; diff --git a/stroom-core-client/src/main/java/stroom/task/client/presenter/TaskProgressUtil.java b/stroom-core-client/src/main/java/stroom/task/client/presenter/TaskProgressUtil.java index 7767cefede2..0e77f2bbc6e 100644 --- a/stroom-core-client/src/main/java/stroom/task/client/presenter/TaskProgressUtil.java +++ b/stroom-core-client/src/main/java/stroom/task/client/presenter/TaskProgressUtil.java @@ -323,10 +323,10 @@ public int compare(final TaskProgress o1, final TaskProgress o2) { int compare = 0; switch (field) { case FindTaskProgressCriteria.FIELD_NAME: - compare = CompareUtil.compareString(o1.getTaskName(), o2.getTaskName()); + compare = CompareUtil.compareStringIgnoreCase(o1.getTaskName(), o2.getTaskName()); break; case FindTaskProgressCriteria.FIELD_USER: - compare = CompareUtil.compareString(o1.getUserName(), o2.getUserName()); + compare = CompareUtil.compareStringIgnoreCase(o1.getUserName(), o2.getUserName()); break; case FindTaskProgressCriteria.FIELD_SUBMIT_TIME: compare = CompareUtil.compareLong(o1.getSubmitTimeMs(), o2.getSubmitTimeMs()); @@ -335,10 +335,10 @@ public int compare(final TaskProgress o1, final TaskProgress o2) { compare = CompareUtil.compareLong(o1.getAgeMs(), o2.getAgeMs()); break; case FindTaskProgressCriteria.FIELD_INFO: - compare = CompareUtil.compareString(o1.getTaskInfo(), o2.getTaskInfo()); + compare = CompareUtil.compareStringIgnoreCase(o1.getTaskInfo(), o2.getTaskInfo()); break; case FindTaskProgressCriteria.FIELD_NODE: - compare = CompareUtil.compareString(o1.getNodeName(), o2.getNodeName()); + compare = CompareUtil.compareStringIgnoreCase(o1.getNodeName(), o2.getNodeName()); break; } diff --git a/stroom-core-client/src/main/java/stroom/task/client/view/TaskManagerViewImpl.java b/stroom-core-client/src/main/java/stroom/task/client/view/TaskManagerViewImpl.java index 60bfcf81782..b927e3fcb7f 100644 --- a/stroom-core-client/src/main/java/stroom/task/client/view/TaskManagerViewImpl.java +++ b/stroom-core-client/src/main/java/stroom/task/client/view/TaskManagerViewImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/stroom-core-client/src/test/java/stroom/util/client/TestParamUtil.java b/stroom-core-client/src/test/java/stroom/util/client/TestParamUtil.java index 1b2a7beb60b..44bd48ac40f 100644 --- a/stroom-core-client/src/test/java/stroom/util/client/TestParamUtil.java +++ b/stroom-core-client/src/test/java/stroom/util/client/TestParamUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import stroom.query.api.v2.Param; import stroom.query.api.v2.ParamUtil; +import stroom.util.shared.string.CIKey; import org.junit.jupiter.api.Test; @@ -66,7 +67,7 @@ void testComplexParse() { @Test void testReplacement() { - Map map = getMap("key1=value1"); + Map map = getMap("key1=value1"); String result = ParamUtil.replaceParameters("this is ${key1}", map); assertThat(result).isEqualTo("this is value1"); @@ -91,13 +92,13 @@ void testReplacement() { assertThat(result).isEqualTo("user1 user2"); } - private Map getMap(final String input) { + private Map getMap(final String input) { final List list = ParamUtil.parse(input); return ParamUtil.createParamMap(list); } private void testKV(String text, String... expectedParams) { - final Map map = getMap(text); + final Map map = getMap(text); assertThat(expectedParams.length > 0).isTrue(); assertThat(expectedParams.length % 2 == 0).isTrue(); @@ -106,7 +107,7 @@ private void testKV(String text, String... expectedParams) { for (int i = 0; i < expectedParams.length; i += 2) { String key = expectedParams[i]; String value = expectedParams[i + 1]; - assertThat(map.get(key)).isEqualTo(value); + assertThat(map.get(CIKey.of(key))).isEqualTo(value); } } } diff --git a/stroom-core-shared/src/main/java/stroom/analytics/shared/ExecutionHistoryFields.java b/stroom-core-shared/src/main/java/stroom/analytics/shared/ExecutionHistoryFields.java index 2cd717977f0..360c5811be8 100644 --- a/stroom-core-shared/src/main/java/stroom/analytics/shared/ExecutionHistoryFields.java +++ b/stroom-core-shared/src/main/java/stroom/analytics/shared/ExecutionHistoryFields.java @@ -1,13 +1,10 @@ package stroom.analytics.shared; public class ExecutionHistoryFields { + public static final String ID = "Id"; public static final String EXECUTION_TIME = "Execution Time"; - public static final String EFFECTIVE_EXECUTION_TIME = "Effective Execution Time"; - public static final String STATUS = "Status"; - public static final String MESSAGE = "Message"; - } diff --git a/stroom-core-shared/src/main/java/stroom/data/shared/DataInfoSection.java b/stroom-core-shared/src/main/java/stroom/data/shared/DataInfoSection.java index 44f31b8df20..0beb13206d8 100644 --- a/stroom-core-shared/src/main/java/stroom/data/shared/DataInfoSection.java +++ b/stroom-core-shared/src/main/java/stroom/data/shared/DataInfoSection.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.data.shared; @@ -31,6 +47,10 @@ public List getEntries() { return entries; } + + // -------------------------------------------------------------------------------- + + @JsonInclude(Include.NON_NULL) public static class Entry { diff --git a/stroom-core-shared/src/main/java/stroom/explorer/shared/ExplorerFields.java b/stroom-core-shared/src/main/java/stroom/explorer/shared/ExplorerFields.java index 117de3c6ad5..6fc4ed879e7 100644 --- a/stroom-core-shared/src/main/java/stroom/explorer/shared/ExplorerFields.java +++ b/stroom-core-shared/src/main/java/stroom/explorer/shared/ExplorerFields.java @@ -1,11 +1,12 @@ package stroom.explorer.shared; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; public class ExplorerFields { public static final String EXPLORER_TYPE = "Explorer"; - public static final QueryField CONTENT = QueryField.createText("Content"); - public static final QueryField TAG = QueryField.createText("Tags"); + public static final QueryField CONTENT = QueryField.createText(CIKey.ofStaticKey("Content"), true); + public static final QueryField TAG = QueryField.createText(CIKey.ofStaticKey("Tags"), true); } diff --git a/stroom-core-shared/src/main/java/stroom/index/shared/IndexShardFields.java b/stroom-core-shared/src/main/java/stroom/index/shared/IndexShardFields.java index 3253a7e8cb9..afee7b24cba 100644 --- a/stroom-core-shared/src/main/java/stroom/index/shared/IndexShardFields.java +++ b/stroom-core-shared/src/main/java/stroom/index/shared/IndexShardFields.java @@ -2,6 +2,7 @@ import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.Arrays; @@ -26,18 +27,26 @@ public class IndexShardFields { public static final String FIELD_NAME_STATUS = "Status"; public static final String FIELD_NAME_LAST_COMMIT = "Last Commit"; - public static final QueryField FIELD_NODE = QueryField.createText(FIELD_NAME_NODE); + public static final QueryField FIELD_NODE = QueryField.createText( + CIKey.ofStaticKey(FIELD_NAME_NODE), true); public static final QueryField FIELD_INDEX = QueryField - .createDocRefByUuid(LuceneIndexDoc.DOCUMENT_TYPE, FIELD_NAME_INDEX); + .createDocRefByUuid(LuceneIndexDoc.DOCUMENT_TYPE, CIKey.ofStaticKey(FIELD_NAME_INDEX)); public static final QueryField FIELD_INDEX_NAME = QueryField.createDocRefByNonUniqueName( - LuceneIndexDoc.DOCUMENT_TYPE, FIELD_NAME_INDEX_NAME); - public static final QueryField FIELD_VOLUME_PATH = QueryField.createText(FIELD_NAME_VOLUME_PATH); - public static final QueryField FIELD_VOLUME_GROUP = QueryField.createText(FIELD_NAME_VOLUME_GROUP); - public static final QueryField FIELD_PARTITION = QueryField.createText(FIELD_NAME_PARTITION); - public static final QueryField FIELD_DOC_COUNT = QueryField.createInteger(FIELD_NAME_DOC_COUNT); - public static final QueryField FIELD_FILE_SIZE = QueryField.createLong(FIELD_NAME_FILE_SIZE); - public static final QueryField FIELD_STATUS = QueryField.createText(FIELD_NAME_STATUS); - public static final QueryField FIELD_LAST_COMMIT = QueryField.createDate(FIELD_NAME_LAST_COMMIT); + LuceneIndexDoc.DOCUMENT_TYPE, CIKey.ofStaticKey(FIELD_NAME_INDEX_NAME)); + public static final QueryField FIELD_VOLUME_PATH = QueryField.createText( + CIKey.ofStaticKey(FIELD_NAME_VOLUME_PATH), true); + public static final QueryField FIELD_VOLUME_GROUP = QueryField.createText( + CIKey.ofStaticKey(FIELD_NAME_VOLUME_GROUP), true); + public static final QueryField FIELD_PARTITION = QueryField.createText( + CIKey.ofStaticKey(FIELD_NAME_PARTITION), true); + public static final QueryField FIELD_DOC_COUNT = QueryField.createInteger( + CIKey.ofStaticKey(FIELD_NAME_DOC_COUNT), true); + public static final QueryField FIELD_FILE_SIZE = QueryField.createLong( + CIKey.ofStaticKey(FIELD_NAME_FILE_SIZE), true); + public static final QueryField FIELD_STATUS = QueryField.createText( + CIKey.ofStaticKey(FIELD_NAME_STATUS), true); + public static final QueryField FIELD_LAST_COMMIT = QueryField.createDate( + CIKey.ofStaticKey(FIELD_NAME_LAST_COMMIT), true); // GWT so no List.of private static final List FIELDS = Arrays.asList( diff --git a/stroom-core-shared/src/main/java/stroom/index/shared/IndexVolumeFields.java b/stroom-core-shared/src/main/java/stroom/index/shared/IndexVolumeFields.java index 074c2cac433..ddbbc08f25f 100644 --- a/stroom-core-shared/src/main/java/stroom/index/shared/IndexVolumeFields.java +++ b/stroom-core-shared/src/main/java/stroom/index/shared/IndexVolumeFields.java @@ -1,6 +1,7 @@ package stroom.index.shared; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.List; @@ -17,11 +18,12 @@ public class IndexVolumeFields { public static final String FIELD_PATH = "Path"; private static final List FIELDS = new ArrayList<>(); - private static final Map FIELD_MAP; + private static final Map FIELD_MAP; - public static final QueryField GROUP_ID = QueryField.createId(FIELD_GROUP_ID); - public static final QueryField NODE_NAME = QueryField.createText(FIELD_NODE_NAME); - public static final QueryField PATH = QueryField.createText(FIELD_PATH); + public static final QueryField GROUP_ID = QueryField.createId(CIKey.ofStaticKey(FIELD_GROUP_ID), true); + public static final QueryField NODE_NAME = QueryField.createText( + CIKey.ofStaticKey(FIELD_NODE_NAME), true); + public static final QueryField PATH = QueryField.createText(CIKey.ofStaticKey(FIELD_PATH), true); // Id's public static final QueryField ID = QueryField.createId(FIELD_ID); @@ -35,14 +37,17 @@ public class IndexVolumeFields { // Id's FIELDS.add(ID); - FIELD_MAP = FIELDS.stream().collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + FIELD_MAP = FIELDS.stream() + .collect(Collectors.toMap( + QueryField::getFldNameAsCIKey, + Function.identity())); } public static List getFields() { return new ArrayList<>(FIELDS); } - public static Map getFieldMap() { + public static Map getFieldMap() { return FIELD_MAP; } } diff --git a/stroom-core-shared/src/main/java/stroom/meta/shared/DataRetentionFields.java b/stroom-core-shared/src/main/java/stroom/meta/shared/DataRetentionFields.java index 847475e27b9..f40e8c9ef54 100644 --- a/stroom-core-shared/src/main/java/stroom/meta/shared/DataRetentionFields.java +++ b/stroom-core-shared/src/main/java/stroom/meta/shared/DataRetentionFields.java @@ -1,6 +1,7 @@ package stroom.meta.shared; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; public final class DataRetentionFields { @@ -8,9 +9,12 @@ public final class DataRetentionFields { public static final String RETENTION_UNTIL = "Until"; public static final String RETENTION_RULE = "Rule"; - public static final QueryField RETENTION_AGE_FIELD = QueryField.createText(RETENTION_AGE, false); - public static final QueryField RETENTION_UNTIL_FIELD = QueryField.createText(RETENTION_UNTIL, false); - public static final QueryField RETENTION_RULE_FIELD = QueryField.createText(RETENTION_RULE, false); + public static final QueryField RETENTION_AGE_FIELD = QueryField.createText( + CIKey.ofStaticKey(RETENTION_AGE), false); + public static final QueryField RETENTION_UNTIL_FIELD = QueryField.createText( + CIKey.ofStaticKey(RETENTION_UNTIL), false); + public static final QueryField RETENTION_RULE_FIELD = QueryField.createText( + CIKey.ofStaticKey(RETENTION_RULE), false); private DataRetentionFields() { } diff --git a/stroom-core-shared/src/main/java/stroom/meta/shared/Meta.java b/stroom-core-shared/src/main/java/stroom/meta/shared/Meta.java index 33626de5dbb..291348b94ab 100644 --- a/stroom-core-shared/src/main/java/stroom/meta/shared/Meta.java +++ b/stroom-core-shared/src/main/java/stroom/meta/shared/Meta.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -207,6 +207,10 @@ public Builder copy() { return new Builder(this); } + + // -------------------------------------------------------------------------------- + + public static final class Builder { private long id; diff --git a/stroom-core-shared/src/main/java/stroom/meta/shared/MetaAttributeMapUtil.java b/stroom-core-shared/src/main/java/stroom/meta/shared/MetaAttributeMapUtil.java index d1efe1b59ab..8eaccbdf6b7 100644 --- a/stroom-core-shared/src/main/java/stroom/meta/shared/MetaAttributeMapUtil.java +++ b/stroom-core-shared/src/main/java/stroom/meta/shared/MetaAttributeMapUtil.java @@ -1,6 +1,23 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.meta.shared; import stroom.docref.DocRef; +import stroom.util.shared.string.CIKey; import java.util.HashMap; import java.util.Map; @@ -14,36 +31,36 @@ private MetaAttributeMapUtil() { /** * Turns a stream attribute map object into a generic map of attributes for use by an expression filter. */ - public static Map createAttributeMap(final Meta meta) { - final Map map = new HashMap<>(); + public static Map createAttributeMap(final Meta meta) { + final Map map = new HashMap<>(); if (meta != null) { // Non grouped fields final String feedName = meta.getFeedName(); if (feedName != null) { - map.put(MetaFields.FEED.getFldName(), feedName); + map.put(MetaFields.FEED.getFldNameAsCIKey(), feedName); } final String pipelineUuid = meta.getPipelineUuid(); if (pipelineUuid != null) { - map.put(MetaFields.PIPELINE.getFldName(), new DocRef("Pipeline", pipelineUuid)); + map.put(MetaFields.PIPELINE.getFldNameAsCIKey(), new DocRef("Pipeline", pipelineUuid)); } if (meta.getStatus() != null) { - map.put(MetaFields.STATUS.getFldName(), meta.getStatus().getDisplayValue()); + map.put(MetaFields.STATUS.getFldNameAsCIKey(), meta.getStatus().getDisplayValue()); } if (meta.getTypeName() != null) { - map.put(MetaFields.TYPE.getFldName(), meta.getTypeName()); + map.put(MetaFields.TYPE.getFldNameAsCIKey(), meta.getTypeName()); } // Id's - map.put(MetaFields.ID.getFldName(), meta.getId()); + map.put(MetaFields.ID.getFldNameAsCIKey(), meta.getId()); if (meta.getParentMetaId() != null) { - map.put(MetaFields.PARENT_ID.getFldName(), meta.getParentMetaId()); + map.put(MetaFields.PARENT_ID.getFldNameAsCIKey(), meta.getParentMetaId()); } // Times - map.put(MetaFields.CREATE_TIME.getFldName(), meta.getCreateMs()); - map.put(MetaFields.EFFECTIVE_TIME.getFldName(), meta.getEffectiveMs()); - map.put(MetaFields.STATUS_TIME.getFldName(), meta.getStatusMs()); + map.put(MetaFields.CREATE_TIME.getFldNameAsCIKey(), meta.getCreateMs()); + map.put(MetaFields.EFFECTIVE_TIME.getFldNameAsCIKey(), meta.getEffectiveMs()); + map.put(MetaFields.STATUS_TIME.getFldNameAsCIKey(), meta.getStatusMs()); // FIELDS.add(META_INTERNAL_PROCESSOR_ID); // FIELDS.add(META_PROCESSOR_FILTER_ID); diff --git a/stroom-core-shared/src/main/java/stroom/meta/shared/MetaFields.java b/stroom-core-shared/src/main/java/stroom/meta/shared/MetaFields.java index 7371dd702d2..9854bf85fda 100644 --- a/stroom-core-shared/src/main/java/stroom/meta/shared/MetaFields.java +++ b/stroom-core-shared/src/main/java/stroom/meta/shared/MetaFields.java @@ -1,8 +1,27 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.meta.shared; import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; +import stroom.feed.shared.FeedDoc; import stroom.pipeline.shared.PipelineDoc; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import java.util.ArrayList; import java.util.List; @@ -25,56 +44,63 @@ public class MetaFields { public static final String FIELD_PARENT_FEED = "Parent Feed"; private static final List FIELDS = new ArrayList<>(); - private static final Map FIELD_MAP; + private static final Map FIELD_MAP; private static final List EXTENDED_FIELDS = new ArrayList<>(); private static final List ALL_FIELDS = new ArrayList<>(); - private static final Map ALL_FIELD_MAP; + private static final Map ALL_FIELD_MAP; + private static final Map ALL_FIELD_NAME_TO_KEY_MAP; // Non grouped fields // Maps to the docref name (which is unique) - public static final QueryField FEED = QueryField.createDocRefByUniqueName("Feed", "Feed"); + public static final QueryField FEED = QueryField.createDocRefByUniqueName( + FeedDoc.DOCUMENT_TYPE, CIKeys.FEED); // Maps to the docref uuid public static final QueryField PIPELINE = QueryField.createDocRefByUuid( PipelineDoc.DOCUMENT_TYPE, - "Pipeline"); + CIKeys.PIPELINE); // Maps to the docref name (which is not unique) public static final QueryField PIPELINE_NAME = QueryField.createDocRefByNonUniqueName( PipelineDoc.DOCUMENT_TYPE, - "Pipeline Name"); + CIKeys.PIPELINE__NAME); - public static final QueryField STATUS = QueryField.createText("Status"); - public static final QueryField TYPE = QueryField.createText("Type"); + public static final QueryField STATUS = QueryField.createText(CIKeys.STATUS, true); + public static final QueryField TYPE = QueryField.createText(CIKeys.TYPE, true); // Id's - public static final QueryField ID = QueryField.createId("Id"); - public static final QueryField META_INTERNAL_PROCESSOR_ID = QueryField.createId("Processor Id"); - public static final QueryField META_PROCESSOR_FILTER_ID = QueryField.createId("Processor Filter Id"); - public static final QueryField META_PROCESSOR_TASK_ID = QueryField.createId("Processor Task Id"); + public static final QueryField ID = QueryField.createId(CIKeys.ID, true); + public static final QueryField META_INTERNAL_PROCESSOR_ID = QueryField.createId( + CIKeys.PROCESSOR__ID, true); + public static final QueryField META_PROCESSOR_FILTER_ID = QueryField.createId( + CIKeys.PROCESSOR__FILTER__ID, true); + public static final QueryField META_PROCESSOR_TASK_ID = QueryField.createId( + CIKeys.PROCESSOR__TASK__ID, true); // Times - public static final QueryField CREATE_TIME = QueryField.createDate("Create Time"); - public static final QueryField EFFECTIVE_TIME = QueryField.createDate("Effective Time"); - public static final QueryField STATUS_TIME = QueryField.createDate("Status Time"); + public static final QueryField CREATE_TIME = QueryField.createDate(CIKeys.CREATE__TIME, true); + public static final QueryField EFFECTIVE_TIME = QueryField.createDate(CIKeys.EFFECTIVE__TIME, true); + public static final QueryField STATUS_TIME = QueryField.createDate(CIKeys.STATUS__TIME, true); // Extended fields. // public static final String NODE = "Node"; - public static final QueryField REC_READ = QueryField.createLong("Read Count"); - public static final QueryField REC_WRITE = QueryField.createLong("Write Count"); - public static final QueryField REC_INFO = QueryField.createLong("Info Count"); - public static final QueryField REC_WARN = QueryField.createLong("Warning Count"); - public static final QueryField REC_ERROR = QueryField.createLong("Error Count"); - public static final QueryField REC_FATAL = QueryField.createLong("Fatal Error Count"); - public static final QueryField DURATION = QueryField.createLong("Duration"); - public static final QueryField FILE_SIZE = QueryField.createLong("File Size"); - public static final QueryField RAW_SIZE = QueryField.createLong("Raw Size"); + public static final QueryField REC_READ = QueryField.createLong(CIKeys.READ__COUNT, true); + public static final QueryField REC_WRITE = QueryField.createLong(CIKeys.WRITE__COUNT, true); + public static final QueryField REC_INFO = QueryField.createLong(CIKeys.INFO__COUNT, true); + public static final QueryField REC_WARN = QueryField.createLong(CIKeys.WARNING__COUNT, true); + public static final QueryField REC_ERROR = QueryField.createLong(CIKeys.ERROR__COUNT, true); + public static final QueryField REC_FATAL = QueryField.createLong(CIKeys.FATAL__ERROR__COUNT, true); + public static final QueryField DURATION = QueryField.createLong(CIKeys.DURATION, true); + public static final QueryField FILE_SIZE = QueryField.createLong(CIKeys.FILE__SIZE, true); + public static final QueryField RAW_SIZE = QueryField.createLong(CIKeys.RAW__SIZE, true); // Parent fields. - public static final QueryField PARENT_ID = QueryField.createId("Parent Id"); - public static final QueryField PARENT_STATUS = QueryField.createText("Parent Status"); - public static final QueryField PARENT_CREATE_TIME = QueryField.createDate("Parent Create Time"); - public static final QueryField PARENT_FEED = QueryField.createDocRefByUniqueName("Feed", FIELD_PARENT_FEED); + public static final QueryField PARENT_ID = QueryField.createId(CIKeys.PARENT__ID, true); + public static final QueryField PARENT_STATUS = QueryField.createText(CIKeys.PARENT__STATUS, true); + public static final QueryField PARENT_CREATE_TIME = QueryField.createDate( + CIKeys.PARENT__CREATE__TIME, true); + public static final QueryField PARENT_FEED = QueryField.createDocRefByUniqueName( + FeedDoc.DOCUMENT_TYPE, CIKeys.PARENT__FEED); static { // Non grouped fields @@ -96,7 +122,10 @@ public class MetaFields { FIELDS.add(EFFECTIVE_TIME); FIELDS.add(STATUS_TIME); - FIELD_MAP = FIELDS.stream().collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + FIELD_MAP = FIELDS.stream() + .collect(Collectors.toMap( + QueryField::getFldNameAsCIKey, + Function.identity())); // Single Items EXTENDED_FIELDS.add(DURATION); @@ -115,14 +144,24 @@ public class MetaFields { ALL_FIELDS.addAll(FIELDS); ALL_FIELDS.addAll(EXTENDED_FIELDS); - ALL_FIELD_MAP = ALL_FIELDS.stream().collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + + ALL_FIELD_MAP = ALL_FIELDS.stream() + .collect(Collectors.toMap( + (QueryField queryField) -> CIKey.of(queryField.getFldName()), + Function.identity())); + + ALL_FIELD_NAME_TO_KEY_MAP = ALL_FIELD_MAP.keySet() + .stream() + .collect(Collectors.toMap( + CIKey::get, + Function.identity())); } public static List getFields() { return new ArrayList<>(FIELDS); } - public static Map getFieldMap() { + public static Map getFieldMap() { return FIELD_MAP; } @@ -130,11 +169,15 @@ public static List getAllFields() { return ALL_FIELDS; } - public static Map getAllFieldMap() { + public static Map getAllFieldMap() { return ALL_FIELD_MAP; } public static List getExtendedFields() { return EXTENDED_FIELDS; } + + public static CIKey createCIKey(final String fieldName) { + return CIKey.of(fieldName, ALL_FIELD_NAME_TO_KEY_MAP); + } } diff --git a/stroom-core-shared/src/main/java/stroom/meta/shared/MetaRow.java b/stroom-core-shared/src/main/java/stroom/meta/shared/MetaRow.java index f2d7c166ce0..73d9c9c6f52 100644 --- a/stroom-core-shared/src/main/java/stroom/meta/shared/MetaRow.java +++ b/stroom-core-shared/src/main/java/stroom/meta/shared/MetaRow.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,6 +30,8 @@ public class MetaRow { private final Meta meta; @JsonProperty private final String pipelineName; + + // Can't use a CIKey keyed map due to GWT @JsonProperty private final Map attributes; @@ -40,6 +42,7 @@ public MetaRow(@JsonProperty("meta") final Meta meta, this.meta = meta; this.pipelineName = pipelineName; this.attributes = attributes; + } public Meta getMeta() { @@ -67,8 +70,8 @@ public boolean equals(final Object o) { return false; } + //noinspection PatternVariableCanBeUsed // Not in GWT land final MetaRow that = (MetaRow) o; - return meta.equals(that.meta); } @@ -79,6 +82,7 @@ public int hashCode() { @Override public String toString() { - return meta.toString(); + return "meta: " + meta + + " - pipeline: '" + pipelineName + '\''; } } diff --git a/stroom-core-shared/src/main/java/stroom/pipeline/shared/ReferenceDataFields.java b/stroom-core-shared/src/main/java/stroom/pipeline/shared/ReferenceDataFields.java index 4d3ae5daa1d..132d88470e9 100644 --- a/stroom-core-shared/src/main/java/stroom/pipeline/shared/ReferenceDataFields.java +++ b/stroom-core-shared/src/main/java/stroom/pipeline/shared/ReferenceDataFields.java @@ -1,9 +1,26 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.pipeline.shared; import stroom.datasource.api.v2.ConditionSet; import stroom.datasource.api.v2.FieldType; import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; +import stroom.util.shared.string.CIKeys; import java.util.Arrays; import java.util.List; @@ -14,58 +31,59 @@ public class ReferenceDataFields { "Searchable", "Reference Data Store", "Reference Data Store (This Node Only)"); + public static final QueryField FEED_NAME_FIELD = QueryField .builder() - .fldName("Feed Name") + .fldName(CIKeys.FEED__NAME) .fldType(FieldType.TEXT) .conditionSet(ConditionSet.REF_DATA_TEXT) .queryable(true) .build(); public static final QueryField KEY_FIELD = QueryField .builder() - .fldName("Key") + .fldName(CIKeys.KEY) .fldType(FieldType.TEXT) .conditionSet(ConditionSet.REF_DATA_TEXT) .queryable(true) .build(); public static final QueryField VALUE_FIELD = QueryField .builder() - .fldName("Value") + .fldName(CIKeys.VALUE) .fldType(FieldType.TEXT) .conditionSet(ConditionSet.REF_DATA_TEXT) .queryable(true) .build(); public static final QueryField VALUE_REF_COUNT_FIELD = QueryField.createInteger( - "Value Reference Count", false); + CIKeys.VALUE__REFERENCE__COUNT, false); public static final QueryField MAP_NAME_FIELD = QueryField .builder() - .fldName("Map Name") + .fldName(CIKeys.MAP__NAME) .fldType(FieldType.TEXT) .conditionSet(ConditionSet.REF_DATA_TEXT) .queryable(true) .build(); public static final QueryField CREATE_TIME_FIELD = QueryField - .createDate("Create Time", true); + .createDate(CIKeys.CREATE__TIME, true); public static final QueryField EFFECTIVE_TIME_FIELD = QueryField - .createDate("Effective Time", true); + .createDate(CIKeys.EFFECTIVE__TIME, true); public static final QueryField LAST_ACCESSED_TIME_FIELD = QueryField - .createDate("Last Accessed Time", true); + .createDate(CIKeys.LAST__ACCESSED__TIME, true); public static final QueryField PIPELINE_FIELD = QueryField .builder() - .fldName("Reference Loader Pipeline") + .fldName(CIKeys.REFERENCE__LOADER__PIPELINE) .fldType(FieldType.DOC_REF) .conditionSet(ConditionSet.REF_DATA_DOC_REF) .docRefType(PipelineDoc.DOCUMENT_TYPE) .queryable(true) .build(); public static final QueryField PROCESSING_STATE_FIELD = QueryField - .createText("Processing State", false); + .createText(CIKeys.PROCESSING__STATE, false); public static final QueryField STREAM_ID_FIELD = QueryField.createId( - "Stream ID", false); + CIKeys.STREAM__ID, false); public static final QueryField PART_NO_FIELD = QueryField.createLong( - "Part Number", false); + CIKeys.PART__NUMBER, false); public static final QueryField PIPELINE_VERSION_FIELD = QueryField.createText( - "Pipeline Version", false); + CIKeys.PIPELINE__VERSION, false); public static final List FIELDS = Arrays.asList( FEED_NAME_FIELD, diff --git a/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorFields.java b/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorFields.java index fac5841b8cf..f8a1b5de499 100644 --- a/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorFields.java +++ b/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorFields.java @@ -1,10 +1,11 @@ package stroom.processor.shared; import stroom.analytics.shared.AnalyticRuleDoc; -import stroom.datasource.api.v2.ConditionSet; import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; import stroom.pipeline.shared.PipelineDoc; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import java.util.ArrayList; import java.util.List; @@ -22,18 +23,18 @@ public class ProcessorFields { .build(); private static final List FIELDS = new ArrayList<>(); - private static final Map ALL_FIELD_MAP; + private static final Map ALL_FIELD_MAP; - public static final QueryField ID = QueryField.createId("Processor Id"); - public static final QueryField PROCESSOR_TYPE = QueryField.createText("Processor Type"); + public static final QueryField ID = QueryField.createId(CIKeys.PROCESSOR__ID, true); + public static final QueryField PROCESSOR_TYPE = QueryField.createText(CIKeys.PROCESSOR__TYPE, true); public static final QueryField PIPELINE = QueryField.createDocRefByUuid( - PipelineDoc.DOCUMENT_TYPE, "Processor Pipeline"); + PipelineDoc.DOCUMENT_TYPE, CIKeys.PROCESSOR__PIPELINE); public static final QueryField ANALYTIC_RULE = QueryField.createDocRefByUuid( - AnalyticRuleDoc.DOCUMENT_TYPE, "Analytic Rule"); - public static final QueryField ENABLED = QueryField.createBoolean("Processor Enabled"); - public static final QueryField DELETED = QueryField.createBoolean("Processor Deleted"); - public static final QueryField UUID = QueryField.createText("Processor UUID"); + AnalyticRuleDoc.DOCUMENT_TYPE, CIKeys.ANALYTIC__RULE); + public static final QueryField ENABLED = QueryField.createBoolean(CIKeys.PROCESSOR__ENABLED, true); + public static final QueryField DELETED = QueryField.createBoolean(CIKeys.PROCESSOR__DELETED, true); + public static final QueryField UUID = QueryField.createText(CIKeys.PROCESSOR__UUID, true); static { FIELDS.add(ID); @@ -44,14 +45,15 @@ public class ProcessorFields { FIELDS.add(DELETED); FIELDS.add(UUID); - ALL_FIELD_MAP = FIELDS.stream().collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + ALL_FIELD_MAP = FIELDS.stream() + .collect(Collectors.toMap(QueryField::getFldNameAsCIKey, Function.identity())); } public static List getFields() { return new ArrayList<>(FIELDS); } - public static Map getAllFieldMap() { + public static Map getAllFieldMap() { return ALL_FIELD_MAP; } } diff --git a/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorFilterFields.java b/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorFilterFields.java index 0a4bf234519..799bdf09063 100644 --- a/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorFilterFields.java +++ b/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorFilterFields.java @@ -3,6 +3,7 @@ import stroom.datasource.api.v2.ConditionSet; import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.List; @@ -22,17 +23,24 @@ public class ProcessorFilterFields { public static final String FIELD_ID = "Id"; private static final List FIELDS = new ArrayList<>(); - private static final Map FIELD_MAP; + private static final Map FIELD_MAP; - public static final QueryField ID = QueryField.createId("Processor Filter Id"); + public static final QueryField ID = QueryField.createId(CIKey.ofStaticKey("Processor Filter Id"), true); // public static final QueryField CREATE_USER = QueryField.createText("Processor Filter Create User"); - public static final QueryField OWNER_UUID = QueryField.createText("Processor Filter Owner User UUID"); - public static final QueryField LAST_POLL_MS = QueryField.createLong("Processor Filter Last Poll Ms"); - public static final QueryField PRIORITY = QueryField.createInteger("Processor Filter Priority"); - public static final QueryField ENABLED = QueryField.createBoolean("Processor Filter Enabled"); - public static final QueryField DELETED = QueryField.createBoolean("Processor Filter Deleted"); - public static final QueryField PROCESSOR_ID = QueryField.createId("Processor Id"); - public static final QueryField UUID = QueryField.createText("Processor Filter UUID"); + public static final QueryField OWNER_UUID = QueryField.createText( + CIKey.ofStaticKey("Processor Filter Owner User UUID"), true); + public static final QueryField LAST_POLL_MS = QueryField.createLong( + CIKey.ofStaticKey("Processor Filter Last Poll Ms"), true); + public static final QueryField PRIORITY = QueryField.createInteger( + CIKey.ofStaticKey("Processor Filter Priority"), true); + public static final QueryField ENABLED = QueryField.createBoolean( + CIKey.ofStaticKey("Processor Filter Enabled"), true); + public static final QueryField DELETED = QueryField.createBoolean( + CIKey.ofStaticKey("Processor Filter Deleted"), true); + public static final QueryField PROCESSOR_ID = QueryField.createId( + CIKey.ofStaticKey("Processor Id"), true); + public static final QueryField UUID = QueryField.createText( + CIKey.ofStaticKey("Processor Filter UUID"), true); public static final QueryField RUN_AS_USER = QueryField .builder() .fldName("Run As User") @@ -51,14 +59,15 @@ public class ProcessorFilterFields { FIELDS.add(UUID); FIELDS.add(RUN_AS_USER); - FIELD_MAP = FIELDS.stream().collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + FIELD_MAP = FIELDS.stream() + .collect(Collectors.toMap(QueryField::getFldNameAsCIKey, Function.identity())); } public static List getFields() { return new ArrayList<>(FIELDS); } - public static Map getFieldMap() { + public static Map getFieldMap() { return FIELD_MAP; } } diff --git a/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorTaskFields.java b/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorTaskFields.java index 5076a846207..d0fcf9aa368 100644 --- a/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorTaskFields.java +++ b/stroom-core-shared/src/main/java/stroom/processor/shared/ProcessorTaskFields.java @@ -1,8 +1,26 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.processor.shared; import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; +import stroom.feed.shared.FeedDoc; import stroom.pipeline.shared.PipelineDoc; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.List; @@ -18,7 +36,7 @@ public class ProcessorTaskFields { "Processor Tasks"); private static final List FIELDS = new ArrayList<>(); - private static final Map FIELD_MAP; + private static final Map FIELD_MAP; public static final String FIELD_ID = "Id"; public static final String FIELD_CREATE_TIME = "Created"; @@ -33,25 +51,36 @@ public class ProcessorTaskFields { public static final String FIELD_NODE = "Node"; public static final String FIELD_POLL_AGE = "Poll Age"; - public static final QueryField CREATE_TIME = QueryField.createDate("Create Time"); - public static final QueryField CREATE_TIME_MS = QueryField.createLong("Create Time Ms"); - public static final QueryField START_TIME = QueryField.createDate("Start Time"); - public static final QueryField START_TIME_MS = QueryField.createLong("Start Time Ms"); - public static final QueryField END_TIME_MS = QueryField.createLong("End Time Ms"); - public static final QueryField END_TIME = QueryField.createDate("End Time"); - public static final QueryField STATUS_TIME_MS = QueryField.createLong("Status Time Ms"); - public static final QueryField STATUS_TIME = QueryField.createDate("Status Time"); - public static final QueryField META_ID = QueryField.createId("Meta Id"); - public static final QueryField NODE_NAME = QueryField.createText("Node"); - public static final QueryField PIPELINE = QueryField.createDocRefByUuid(PipelineDoc.DOCUMENT_TYPE, FIELD_PIPELINE); + public static final QueryField CREATE_TIME = QueryField.createDate( + CIKey.ofStaticKey("Create Time"), true); + public static final QueryField CREATE_TIME_MS = QueryField.createLong( + CIKey.ofStaticKey("Create Time Ms"), true); + public static final QueryField START_TIME = QueryField.createDate(CIKey.ofStaticKey("Start Time"), true); + public static final QueryField START_TIME_MS = QueryField.createLong( + CIKey.ofStaticKey("Start Time Ms"), true); + public static final QueryField END_TIME_MS = QueryField.createLong( + CIKey.ofStaticKey("End Time Ms"), true); + public static final QueryField END_TIME = QueryField.createDate(CIKey.ofStaticKey("End Time"), true); + public static final QueryField STATUS_TIME_MS = QueryField.createLong( + CIKey.ofStaticKey("Status Time Ms"), true); + public static final QueryField STATUS_TIME = QueryField.createDate( + CIKey.ofStaticKey("Status Time"), true); + public static final QueryField META_ID = QueryField.createId(CIKey.ofStaticKey("Meta Id"), true); + public static final QueryField NODE_NAME = QueryField.createText(CIKey.ofStaticKey(FIELD_NODE), true); + public static final QueryField PIPELINE = QueryField.createDocRefByUuid( + PipelineDoc.DOCUMENT_TYPE, CIKey.ofStaticKey(FIELD_PIPELINE)); public static final QueryField PIPELINE_NAME = QueryField.createDocRefByNonUniqueName( - PipelineDoc.DOCUMENT_TYPE, FIELD_PIPELINE_NAME); - public static final QueryField PROCESSOR_FILTER_ID = QueryField.createId("Processor Filter Id"); - public static final QueryField PROCESSOR_FILTER_PRIORITY = QueryField.createLong("Processor Filter Priority"); - public static final QueryField PROCESSOR_ID = QueryField.createId("Processor Id"); - public static final QueryField FEED = QueryField.createDocRefByUniqueName("Feed", "Feed"); - public static final QueryField STATUS = QueryField.createText("Status"); - public static final QueryField TASK_ID = QueryField.createId("Task Id"); + PipelineDoc.DOCUMENT_TYPE, CIKey.ofStaticKey(FIELD_PIPELINE_NAME)); + public static final QueryField PROCESSOR_FILTER_ID = QueryField.createId(CIKey.ofStaticKey("Processor Filter Id"), + true); + public static final QueryField PROCESSOR_FILTER_PRIORITY = QueryField.createLong(CIKey.ofStaticKey( + "Processor Filter Priority"), true); + public static final QueryField PROCESSOR_ID = QueryField.createId( + CIKey.ofStaticKey("Processor Id"), true); + public static final QueryField FEED = QueryField.createDocRefByUniqueName(FeedDoc.DOCUMENT_TYPE, + CIKey.ofStaticKey(FIELD_FEED)); + public static final QueryField STATUS = QueryField.createText(CIKey.ofStaticKey(FIELD_STATUS), true); + public static final QueryField TASK_ID = QueryField.createId(CIKey.ofStaticKey("Task Id"), true); static { FIELDS.add(CREATE_TIME); @@ -72,15 +101,18 @@ public class ProcessorTaskFields { FIELDS.add(FEED); FIELDS.add(STATUS); FIELDS.add(TASK_ID); + FIELD_MAP = FIELDS.stream() - .collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + .collect(Collectors.toMap( + QueryField::getFldNameAsCIKey, + Function.identity())); } public static List getFields() { return new ArrayList<>(FIELDS); } - public static Map getFieldMap() { + public static Map getFieldMap() { return FIELD_MAP; } } diff --git a/stroom-core-shared/src/main/java/stroom/security/shared/DocumentPermissionFields.java b/stroom-core-shared/src/main/java/stroom/security/shared/DocumentPermissionFields.java index 1a4da5b56d7..76747562759 100644 --- a/stroom-core-shared/src/main/java/stroom/security/shared/DocumentPermissionFields.java +++ b/stroom-core-shared/src/main/java/stroom/security/shared/DocumentPermissionFields.java @@ -4,6 +4,8 @@ import stroom.datasource.api.v2.FieldType; import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import java.util.ArrayList; import java.util.List; @@ -21,60 +23,60 @@ public class DocumentPermissionFields { .build(); private static final List FIELDS = new ArrayList<>(); - private static final Map ALL_FIELD_MAP; + private static final Map ALL_FIELD_MAP; public static final QueryField DOCUMENT = QueryField .builder() - .fldName("Document") + .fldName(CIKey.ofStaticKey("Document")) .fldType(FieldType.DOC_REF) .conditionSet(ConditionSet.DOC_DOC_IS) .queryable(true) .build(); public static final QueryField CHILDREN = QueryField .builder() - .fldName("Children") + .fldName(CIKey.ofStaticKey("Children")) .fldType(FieldType.DOC_REF) .conditionSet(ConditionSet.DOC_DOC_OF) .queryable(true) .build(); public static final QueryField DESCENDANTS = QueryField .builder() - .fldName("Descendants") + .fldName(CIKey.ofStaticKey("Descendants")) .fldType(FieldType.DOC_REF) .conditionSet(ConditionSet.DOC_DOC_OF) .queryable(true) .build(); public static final QueryField USER = QueryField .builder() - .fldName("User") + .fldName(CIKeys.USER) .fldType(FieldType.USER_REF) .conditionSet(ConditionSet.DOC_USER_IS) .queryable(true) .build(); public static final QueryField DOCUMENT_TYPE = QueryField .builder() - .fldName("DocumentType") + .fldName(CIKey.ofStaticKey("DocumentType")) .fldType(FieldType.TEXT) .conditionSet(ConditionSet.DEFAULT_TEXT) .queryable(true) .build(); public static final QueryField DOCUMENT_NAME = QueryField .builder() - .fldName("DocumentName") + .fldName(CIKey.ofStaticKey("DocumentName")) .fldType(FieldType.TEXT) .conditionSet(ConditionSet.DEFAULT_TEXT) .queryable(true) .build(); public static final QueryField DOCUMENT_UUID = QueryField .builder() - .fldName("DocumentUUID") + .fldName(CIKey.ofStaticKey("DocumentUUID")) .fldType(FieldType.TEXT) .conditionSet(ConditionSet.DEFAULT_TEXT) .queryable(true) .build(); public static final QueryField DOCUMENT_TAG = QueryField .builder() - .fldName("DocumentTag") + .fldName(CIKey.ofStaticKey("DocumentTag")) .fldType(FieldType.TEXT) .conditionSet(ConditionSet.DEFAULT_TEXT) .queryable(true) @@ -90,14 +92,15 @@ public class DocumentPermissionFields { FIELDS.add(DOCUMENT_TAG); FIELDS.add(USER); - ALL_FIELD_MAP = FIELDS.stream().collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + ALL_FIELD_MAP = FIELDS.stream() + .collect(Collectors.toMap(QueryField::getFldNameAsCIKey, Function.identity())); } public static List getFields() { return new ArrayList<>(FIELDS); } - public static Map getAllFieldMap() { + public static Map getAllFieldMap() { return ALL_FIELD_MAP; } } diff --git a/stroom-core-shared/src/main/java/stroom/security/shared/QuickFilterExpressionParser.java b/stroom-core-shared/src/main/java/stroom/security/shared/QuickFilterExpressionParser.java index f4bfef08e64..b93170b114d 100644 --- a/stroom-core-shared/src/main/java/stroom/security/shared/QuickFilterExpressionParser.java +++ b/stroom-core-shared/src/main/java/stroom/security/shared/QuickFilterExpressionParser.java @@ -6,6 +6,7 @@ import stroom.query.api.v2.ExpressionOperator.Op; import stroom.query.api.v2.ExpressionTerm.Condition; import stroom.util.shared.GwtNullSafe; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.List; @@ -38,7 +39,7 @@ private QuickFilterExpressionParser() { public static ExpressionOperator parse(final String userInput, final Set defaultFields, - final Map fieldMap) { + final Map fieldMap) { // user input like 'vent type:pipe' or just 'vent' @@ -54,7 +55,7 @@ public static ExpressionOperator parse(final String userInput, private static void extractMatchTokens(final String userInput, final Set defaultFields, - final Map fieldMap, + final Map fieldMap, final ExpressionOperator.Builder builder) { final List parts = splitInput(userInput); for (final String part : parts) { @@ -63,7 +64,7 @@ private static void extractMatchTokens(final String userInput, final String[] subParts = part.split(QUALIFIER_DELIMITER_STR); if (part.endsWith(QUALIFIER_DELIMITER_STR)) { final String fieldName = subParts[0]; - final QueryField field = fieldMap.get(fieldName); + final QueryField field = fieldMap.get(CIKey.of(fieldName)); if (field == null) { throw new RuntimeException("Unknown qualifier '" + fieldName + "'. Valid qualifiers: " + fieldMap.keySet()); @@ -74,7 +75,7 @@ private static void extractMatchTokens(final String userInput, throw new RuntimeException("Invalid token " + part); } else { final String fieldName = subParts[0]; - final QueryField field = fieldMap.get(fieldName); + final QueryField field = fieldMap.get(CIKey.of(fieldName)); if (field == null) { throw new RuntimeException("Unknown qualifier '" + fieldName + "'. Valid qualifiers: " + fieldMap.keySet()); diff --git a/stroom-core-shared/src/main/java/stroom/security/shared/UserFields.java b/stroom-core-shared/src/main/java/stroom/security/shared/UserFields.java index 25eef4e1084..c080ed2258d 100644 --- a/stroom-core-shared/src/main/java/stroom/security/shared/UserFields.java +++ b/stroom-core-shared/src/main/java/stroom/security/shared/UserFields.java @@ -4,13 +4,15 @@ import stroom.datasource.api.v2.FieldType; import stroom.datasource.api.v2.QueryField; import stroom.util.shared.filter.FilterFieldDefinition; +import stroom.util.shared.string.CIKey; -import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class UserFields { @@ -19,20 +21,23 @@ public class UserFields { public static final String FIELD_DISPLAY_NAME = "display"; public static final String FIELD_FULL_NAME = "full"; - public static final QueryField IS_GROUP = QueryField.createBoolean(FIELD_IS_GROUP); - public static final QueryField NAME = QueryField.createText(FIELD_NAME); - public static final QueryField DISPLAY_NAME = QueryField.createText(FIELD_DISPLAY_NAME); - public static final QueryField FULL_NAME = QueryField.createText(FIELD_FULL_NAME); + public static final QueryField IS_GROUP = QueryField.createBoolean( + CIKey.ofStaticKey(FIELD_IS_GROUP), true); + public static final QueryField NAME = QueryField.createText(CIKey.ofStaticKey(FIELD_NAME), true); + public static final QueryField DISPLAY_NAME = QueryField.createText( + CIKey.ofStaticKey(FIELD_DISPLAY_NAME), true); + public static final QueryField FULL_NAME = QueryField.createText( + CIKey.ofStaticKey(FIELD_FULL_NAME), true); public static final QueryField GROUP_CONTAINS = QueryField .builder() - .fldName("GroupContains") + .fldName(CIKey.ofStaticKey("GroupContains")) .fldType(FieldType.USER_REF) .conditionSet(ConditionSet.DEFAULT_TEXT) .queryable(true) .build(); public static final QueryField PARENT_GROUP = QueryField .builder() - .fldName("ParentGroup") + .fldName(CIKey.ofStaticKey("ParentGroup")) .fldType(FieldType.USER_REF) .conditionSet(ConditionSet.DEFAULT_TEXT) .queryable(true) @@ -45,25 +50,24 @@ public class UserFields { public static final FilterFieldDefinition FIELD_DEF_FULL_NAME = FilterFieldDefinition.qualifiedField( FIELD_FULL_NAME); - public static final List FILTER_FIELD_DEFINITIONS = Arrays.asList( + public static final List FILTER_FIELD_DEFINITIONS = List.of( FIELD_DEF_IS_GROUP, FIELD_DEF_NAME, FIELD_DEF_DISPLAY_NAME, FIELD_DEF_FULL_NAME); + public static final Set DEFAULT_FIELDS = new HashSet<>(List.of( + DISPLAY_NAME, + NAME)); - public static final Set DEFAULT_FIELDS = new HashSet<>(); - public static final Map ALL_FIELD_MAP = new HashMap<>(); - - static { - DEFAULT_FIELDS.add(DISPLAY_NAME); - DEFAULT_FIELDS.add(NAME); - - ALL_FIELD_MAP.put(IS_GROUP.getFldName(), IS_GROUP); - ALL_FIELD_MAP.put(NAME.getFldName(), NAME); - ALL_FIELD_MAP.put(DISPLAY_NAME.getFldName(), DISPLAY_NAME); - ALL_FIELD_MAP.put(FULL_NAME.getFldName(), FULL_NAME); - ALL_FIELD_MAP.put(GROUP_CONTAINS.getFldName(), GROUP_CONTAINS); - ALL_FIELD_MAP.put(PARENT_GROUP.getFldName(), PARENT_GROUP); - } + public static final Map ALL_FIELD_MAP = Stream.of( + IS_GROUP, + NAME, + DISPLAY_NAME, + FULL_NAME, + GROUP_CONTAINS, + PARENT_GROUP) + .collect(Collectors.toMap( + QueryField::getFldNameAsCIKey, + Function.identity())); } diff --git a/stroom-core/src/main/java/stroom/core/receive/ReceiveDataRequestHandler.java b/stroom-core/src/main/java/stroom/core/receive/ReceiveDataRequestHandler.java index ccbbc177f1f..3d23b5a241e 100755 --- a/stroom-core/src/main/java/stroom/core/receive/ReceiveDataRequestHandler.java +++ b/stroom-core/src/main/java/stroom/core/receive/ReceiveDataRequestHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.core.receive; @@ -21,6 +20,7 @@ import stroom.meta.api.AttributeMapUtil; import stroom.meta.api.MetaService; import stroom.meta.api.StandardHeaderArguments; +import stroom.pipeline.writer.CSVFormatter; import stroom.proxy.StroomStatusCode; import stroom.receive.common.AttributeMapFilter; import stroom.receive.common.AttributeMapValidator; @@ -45,7 +45,6 @@ import java.io.IOException; import java.io.InputStream; import java.time.Instant; -import java.util.List; import java.util.function.Consumer; /** @@ -140,23 +139,8 @@ private void logSuccess(final StroomStreamStatus stroomStreamStatus) { private void debug(final String message, final AttributeMap attributeMap) { if (LOGGER.isDebugEnabled()) { - final List keys = attributeMap - .keySet() - .stream() - .sorted() - .toList(); - final StringBuilder sb = new StringBuilder(); - keys.forEach(key -> { - sb.append(key); - sb.append("="); - sb.append(attributeMap.get(key)); - sb.append(","); - }); - if (!sb.isEmpty()) { - sb.setLength(sb.length() - 1); - } - - LOGGER.debug(message + " (" + sb + ")"); + final String attrs = CSVFormatter.format(attributeMap); + LOGGER.debug(message + " (" + attrs + ")"); } } } diff --git a/stroom-core/src/main/java/stroom/core/tools/BenchmarkDataFeed.java b/stroom-core/src/main/java/stroom/core/tools/BenchmarkDataFeed.java index 76de07d048c..08bfa9d77c3 100644 --- a/stroom-core/src/main/java/stroom/core/tools/BenchmarkDataFeed.java +++ b/stroom-core/src/main/java/stroom/core/tools/BenchmarkDataFeed.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import stroom.util.date.DateUtil; import stroom.util.io.StreamUtil; import stroom.util.shared.ModelStringUtil; +import stroom.util.shared.string.CIKey; import stroom.util.thread.CustomThreadFactory; import stroom.util.thread.StroomThreadGroup; @@ -62,7 +63,7 @@ public class BenchmarkDataFeed { public static void main(final String[] args) throws IOException { final BenchmarkDataFeed benchmarkDataFeed = new BenchmarkDataFeed(); - final Map map = ArgsUtil.parse(args); + final Map map = ArgsUtil.parse(args); benchmarkDataFeed.setOptionalArgs(map); benchmarkDataFeed.run(); } @@ -81,7 +82,7 @@ private void logDebug(final String msg) { } } - public void setOptionalArgs(final Map args) { + public void setOptionalArgs(final Map args) { batchSize = readOptionalArgument(args, "BatchSize", batchSize); batchFileSize = readOptionalArgument(args, "BatchFileSize", batchFileSize); serverUrl = readOptionalArgument(args, "ServerUrl", serverUrl); @@ -91,38 +92,42 @@ public void setOptionalArgs(final Map args) { debug = readOptionalArgument(args, "Debug", debug); } - private String readOptionalArgument(final Map args, final String name, final String defValue) { - if (!args.containsKey(name)) { + private String readOptionalArgument(final Map args, final String name, final String defValue) { + final CIKey key = CIKey.of(name); + if (!args.containsKey(key)) { return defValue; } - final String val = args.get(name); + final String val = args.get(key); log("Set " + name + "=" + val); return val; } - private int readOptionalArgument(final Map args, final String name, final int defValue) { - if (!args.containsKey(name)) { + private int readOptionalArgument(final Map args, final String name, final int defValue) { + final CIKey key = CIKey.of(name); + if (!args.containsKey(key)) { return defValue; } - final int val = Integer.valueOf(args.get(name)); + final int val = Integer.parseInt(args.get(key)); log("Set " + name + "=" + val); return val; } - private long readOptionalArgument(final Map args, final String name, final long defValue) { - if (!args.containsKey(name)) { + private long readOptionalArgument(final Map args, final String name, final long defValue) { + final CIKey key = CIKey.of(name); + if (!args.containsKey(key)) { return defValue; } - final long val = Long.valueOf(args.get(name)); + final long val = Long.parseLong(args.get(key)); log("Set " + name + "=" + val); return val; } - private boolean readOptionalArgument(final Map args, final String name, final boolean defValue) { - if (!args.containsKey(name)) { + private boolean readOptionalArgument(final Map args, final String name, final boolean defValue) { + final CIKey key = CIKey.of(name); + if (!args.containsKey(key)) { return defValue; } - final boolean val = Boolean.valueOf(args.get(name)); + final boolean val = Boolean.parseBoolean(args.get(key)); log("Set " + name + "=" + val); return val; } @@ -130,7 +135,12 @@ private boolean readOptionalArgument(final Map args, final Strin private byte[] buildSampleData() { final StringBuilder builder = new StringBuilder(); for (int i = 1; i < 1000; i++) { - builder.append("BenchmarkDataFeed," + i + "," + DateUtil.createNormalDateTimeString() + "\n"); + builder.append("BenchmarkDataFeed,") + .append(i) + .append(",") + .append(DateUtil.createNormalDateTimeString()) + .append( + "\n"); } return builder.toString().getBytes(StreamUtil.DEFAULT_CHARSET); @@ -382,7 +392,7 @@ public void processCommand(final String line) { final String upperLine = line.toUpperCase(); if (upperLine.startsWith("SET ")) { final String[] args = line.substring(4).split(" "); - final Map map = ArgsUtil.parse(args); + final Map map = ArgsUtil.parse(args); setOptionalArgs(map); } if (upperLine.startsWith("START")) { @@ -396,7 +406,7 @@ public void processCommand(final String line) { System.exit(0); } - if (upperLine.equals("") || upperLine.startsWith("STATUS")) { + if (upperLine.isEmpty() || upperLine.startsWith("STATUS")) { statusConfig(); statusBatch(); } diff --git a/stroom-core/src/main/java/stroom/core/tools/MigrationTool.java b/stroom-core/src/main/java/stroom/core/tools/MigrationTool.java index 535ed1ae484..18ae8dec4b9 100644 --- a/stroom-core/src/main/java/stroom/core/tools/MigrationTool.java +++ b/stroom-core/src/main/java/stroom/core/tools/MigrationTool.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import stroom.util.ArgsUtil; import stroom.util.io.StreamUtil; +import stroom.util.shared.string.CIKey; import java.io.IOException; import java.io.InputStream; @@ -41,15 +42,15 @@ public class MigrationTool { public static void main(final String[] args) throws SQLException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, ClassNotFoundException, IOException { - final Map map = ArgsUtil.parse(args); - - final String url = map.get("jdbcDriverUrl"); - String clazz = map.get("jdbcDriverClassName"); - final String username = map.get("username"); - final String password = map.get("password"); - final String script = map.get("script"); - ignoreError = Boolean.TRUE.toString().equalsIgnoreCase(map.get("ignoreError")); - update = !Boolean.FALSE.toString().equalsIgnoreCase(map.get("update")); + final Map map = ArgsUtil.parse(args); + + final String url = map.get(CIKey.of("jdbcDriverUrl")); + String clazz = map.get(CIKey.of("jdbcDriverClassName")); + final String username = map.get(CIKey.of("username")); + final String password = map.get(CIKey.of("password")); + final String script = map.get(CIKey.of("script")); + ignoreError = Boolean.TRUE.toString().equalsIgnoreCase(map.get(CIKey.of("ignoreError"))); + update = !Boolean.FALSE.toString().equalsIgnoreCase(map.get(CIKey.of("update"))); if (clazz == null) { clazz = "com.mysql.cj.jdbc.Driver"; diff --git a/stroom-core/src/main/java/stroom/core/tools/StreamRestoreTool.java b/stroom-core/src/main/java/stroom/core/tools/StreamRestoreTool.java index 7636abce14d..31ffd5ff9f4 100644 --- a/stroom-core/src/main/java/stroom/core/tools/StreamRestoreTool.java +++ b/stroom-core/src/main/java/stroom/core/tools/StreamRestoreTool.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -44,6 +44,9 @@ import java.util.Map; import java.util.StringTokenizer; +// TODO This would seem to be redundant or at least very out of date code. +// It is currently not functional as the meat of it is commented out and out of date +// with respect to the DB schema. public class StreamRestoreTool extends DatabaseTool { private static final int KEY_PAD = 30; @@ -332,8 +335,8 @@ private void sort(final List list) { }); } - private Map readAttributes(final String line, final String streamType, final String feedId) { - final HashMap rtnMap = new HashMap<>(); + private AttributeMap readAttributes(final String line, final String streamType, final String feedId) { + final AttributeMap rtnMap = new AttributeMap(); final StringTokenizer stringTokenizer = new StringTokenizer(line, "/"); final StringBuilder volumePath = new StringBuilder(); @@ -394,8 +397,8 @@ private String getTime(final Path file, final String datePart) { return time; } - private Map readManifestAttributes(final String rootFile) { - final Map rtnMap = new HashMap<>(); + private AttributeMap readManifestAttributes(final String rootFile) { + final AttributeMap rtnMap = new AttributeMap(); final Path manifest = Paths.get(rootFile.substring(0, rootFile.lastIndexOf(".")) + ".mf.dat"); if (Files.isRegularFile(manifest)) { final AttributeMap attributeMap = new AttributeMap(); @@ -421,7 +424,7 @@ private void processStreamTypeFeed(final String fileName, long nextLog = System.currentTimeMillis() + 10000; while ((line = reader.readLine()) != null) { - final Map streamAttributes = readAttributes(line, processStreamType, processFeedId); + final AttributeMap streamAttributes = readAttributes(line, processStreamType, processFeedId); lineCount++; if (System.currentTimeMillis() > nextLog) { diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/DashboardStoreImpl.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/DashboardStoreImpl.java index e33fb33dc11..ecb0e7d4330 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/DashboardStoreImpl.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/DashboardStoreImpl.java @@ -343,8 +343,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/SearchRequestMapper.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/SearchRequestMapper.java index 09eb609cb1f..e77fc42ce04 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/SearchRequestMapper.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/SearchRequestMapper.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.dashboard.impl; @@ -41,6 +40,7 @@ import stroom.query.api.v2.SearchRequest; import stroom.query.api.v2.Sort.SortDirection; import stroom.query.api.v2.TableSettings; +import stroom.util.NullSafe; import stroom.util.json.JsonUtil; import stroom.visualisation.shared.VisualisationDoc; @@ -92,7 +92,7 @@ private Query mapQuery(final DashboardSearchRequest searchRequest) { Param searchExpressionParam = null; List params = null; - if (searchRequest.getSearch().getParams() != null && searchRequest.getSearch().getParams().size() > 0) { + if (!NullSafe.isEmptyCollection(searchRequest.getSearch().getParams())) { params = new ArrayList<>(searchRequest.getSearch().getParams().size()); for (final Param param : searchRequest.getSearch().getParams()) { if (EXPRESSION_JSON_PARAM_KEY.equals(param.getKey())) { diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/script/ScriptStoreImpl.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/script/ScriptStoreImpl.java index 00c6a52520b..dc892b1ac8f 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/script/ScriptStoreImpl.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/script/ScriptStoreImpl.java @@ -209,8 +209,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/vis/VisSettings.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/vis/VisSettings.java index a00ceb6c8c3..b76f4d367b4 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/vis/VisSettings.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/vis/VisSettings.java @@ -37,6 +37,10 @@ public Tab[] getTabs() { return tabs; } + + // -------------------------------------------------------------------------------- + + public static class Limit { private String enabled; @@ -57,6 +61,10 @@ public String getSize() { } } + + // -------------------------------------------------------------------------------- + + public static class Sort { private String enabled; @@ -83,6 +91,10 @@ public String getDirection() { } } + + // -------------------------------------------------------------------------------- + + public static class Field { private String id; @@ -112,6 +124,10 @@ public Sort getSort() { } } + + // -------------------------------------------------------------------------------- + + public static class Values { private Field[] fields; @@ -137,6 +153,10 @@ public Limit getLimit() { } } + + // -------------------------------------------------------------------------------- + + public static class Nest { private Field key; @@ -180,6 +200,10 @@ public Values getValues() { } } + + // -------------------------------------------------------------------------------- + + public static class Structure { private Nest nest; @@ -205,6 +229,10 @@ public Values getValues() { } } + + // -------------------------------------------------------------------------------- + + public static class Data { private Structure structure; @@ -223,6 +251,10 @@ public Structure getStructure() { } } + + // -------------------------------------------------------------------------------- + + public static class Control { private String id; @@ -261,6 +293,10 @@ public String getDefaultValue() { } } + + // -------------------------------------------------------------------------------- + + public static class Tab { private String name; diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationControlIds.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationControlIds.java new file mode 100644 index 00000000000..bb7c163e1a8 --- /dev/null +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationControlIds.java @@ -0,0 +1,13 @@ +package stroom.dashboard.impl.visualisation; + +import stroom.util.shared.string.CIKey; + +public class VisualisationControlIds { + + public static final CIKey FIELD = CIKey.ofStaticKey("field"); + public static final CIKey X = CIKey.ofStaticKey("x"); + public static final CIKey Y = CIKey.ofStaticKey("y"); + + private VisualisationControlIds() { + } +} diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationStoreImpl.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationStoreImpl.java index ccc6f136ea7..38d64ab5fdc 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationStoreImpl.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationStoreImpl.java @@ -196,8 +196,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationTokenConsumerImpl.java b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationTokenConsumerImpl.java index be74ebf7460..de52e0044f8 100644 --- a/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationTokenConsumerImpl.java +++ b/stroom-dashboard/stroom-dashboard-impl/src/main/java/stroom/dashboard/impl/visualisation/VisualisationTokenConsumerImpl.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.dashboard.impl.visualisation; import stroom.dashboard.impl.VisField; @@ -27,6 +43,7 @@ import stroom.query.language.token.TokenType; import stroom.util.NullSafe; import stroom.util.json.JsonUtil; +import stroom.util.shared.string.CIKey; import stroom.visualisation.shared.VisualisationDoc; import jakarta.inject.Inject; @@ -174,11 +191,11 @@ private Map getVisParameters(final List children, final String visName, final Controls controls, final TableSettings parentTableSettings) { - final Map columnMap = NullSafe - .list(parentTableSettings - .getColumns()) - .stream() - .collect(Collectors.toMap(Column::getName, Function.identity())); + + // TODO This will error if we get >1 col with the same name. + // Need to change stroomQL so it prevents multiple columns in the select with the same name. + final Map columnMap = NullSafe.stream(parentTableSettings.getColumns()) + .collect(Collectors.toMap(Column::getNameAsCIKey, Function.identity())); final Map params = new HashMap<>(); for (int i = 0; i < children.size(); i++) { @@ -225,7 +242,7 @@ private Map getVisParameters(final List children, final String columnName = t.getUnescapedText(); // Validate the column name. - final Column column = columnMap.get(columnName); + final Column column = columnMap.get(CIKey.of(columnName)); if (column == null) { throw new TokenException(t, "Unable to find selected column: " + columnName); } @@ -251,8 +268,11 @@ private Map getVisParameters(final List children, } } - if (controlId != null && controlValue != null) { - params.put(controlId, controlValue); + // Need to use resolvedControlId as that is from the will be the correct case + // as controlId came from the user so could be any case + final String resolvedControlId = control.getId(); + if (resolvedControlId != null && controlValue != null) { + params.put(resolvedControlId, controlValue); } } return params; @@ -436,9 +456,14 @@ private VisLimit mapVisLimit(final VisSettings.Limit limit, final SettingResolve return null; } + + // -------------------------------------------------------------------------------- + + private static class Controls { - private final Map controls = new HashMap<>(); + // column id => control + private final Map controlsById = new HashMap<>(); public Controls(final VisSettings visSettings) { // Create a map of controls. @@ -446,9 +471,8 @@ public Controls(final VisSettings visSettings) { for (final Tab tab : visSettings.getTabs()) { if (tab.getControls() != null) { for (final Control control : tab.getControls()) { - if (control != null && control.getId() != null) { - controls.put(control.getId(), control); - } + NullSafe.consume(control, Control::getId, id -> + controlsById.put(CIKey.of(id), control)); } } } @@ -456,10 +480,14 @@ public Controls(final VisSettings visSettings) { } public Control getControl(final String controlId) { - return controls.get(controlId); + return controlsById.get(CIKey.of(controlId)); } } + + // -------------------------------------------------------------------------------- + + private static class SettingResolver { private final Controls controls; diff --git a/stroom-dashboard/stroom-storedquery-impl-db/src/main/java/stroom/storedquery/impl/db/StoredQueryDaoImpl.java b/stroom-dashboard/stroom-storedquery-impl-db/src/main/java/stroom/storedquery/impl/db/StoredQueryDaoImpl.java index 97142b9dd15..c0dc1d2cced 100644 --- a/stroom-dashboard/stroom-storedquery-impl-db/src/main/java/stroom/storedquery/impl/db/StoredQueryDaoImpl.java +++ b/stroom-dashboard/stroom-storedquery-impl-db/src/main/java/stroom/storedquery/impl/db/StoredQueryDaoImpl.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.storedquery.impl.db; import stroom.dashboard.shared.FindStoredQueryCriteria; @@ -9,6 +25,7 @@ import stroom.util.exception.DataChangedException; import stroom.util.shared.ResultPage; import stroom.util.shared.UserRef; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -34,7 +51,7 @@ class StoredQueryDaoImpl implements StoredQueryDao { private static final Logger LOGGER = LoggerFactory.getLogger(stroom.storedquery.impl.StoredQueryDao.class); - private static final Map> FIELD_MAP = Map.of( + private static final Map> FIELD_MAP = CIKey.mapOf( FindStoredQueryCriteria.FIELD_ID, QUERY.ID, FindStoredQueryCriteria.FIELD_NAME, QUERY.NAME, FindStoredQueryCriteria.FIELD_TIME, QUERY.CREATE_TIME_MS); diff --git a/stroom-data/stroom-data-retention-impl/src/main/java/stroom/data/retention/impl/DataRetentionRulesServiceImpl.java b/stroom-data/stroom-data-retention-impl/src/main/java/stroom/data/retention/impl/DataRetentionRulesServiceImpl.java index ae7b15abf64..422643f8cba 100644 --- a/stroom-data/stroom-data-retention-impl/src/main/java/stroom/data/retention/impl/DataRetentionRulesServiceImpl.java +++ b/stroom-data/stroom-data-retention-impl/src/main/java/stroom/data/retention/impl/DataRetentionRulesServiceImpl.java @@ -254,8 +254,10 @@ public DataRetentionRules getOrCreate() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List name, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(name, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-data/stroom-data-store-api/src/main/java/stroom/data/store/api/DataService.java b/stroom-data/stroom-data-store-api/src/main/java/stroom/data/store/api/DataService.java index f2f22608343..b66c69c823c 100644 --- a/stroom-data/stroom-data-store-api/src/main/java/stroom/data/store/api/DataService.java +++ b/stroom-data/stroom-data-store-api/src/main/java/stroom/data/store/api/DataService.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import stroom.pipeline.shared.FetchDataRequest; import stroom.util.shared.ResourceGeneration; import stroom.util.shared.ResourceKey; +import stroom.util.shared.string.CIKey; import java.util.List; import java.util.Map; @@ -34,7 +35,7 @@ public interface DataService { ResourceKey upload(UploadDataRequest request); - Map metaAttributes(long id); + Map metaAttributes(long id); List info(long id); diff --git a/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FileMetaGrep.java b/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FileMetaGrep.java index 45d3a372091..52634b07f36 100644 --- a/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FileMetaGrep.java +++ b/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FileMetaGrep.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import stroom.util.AbstractCommandLineTool; import stroom.util.ArgsUtil; import stroom.util.io.AbstractFileVisitor; +import stroom.util.shared.string.CIKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,14 +39,14 @@ class FileMetaGrep extends AbstractCommandLineTool { private static final Logger LOGGER = LoggerFactory.getLogger(FileMetaGrep.class); - private final Map matchMap; + private final Map matchMap; private String[] repoPathParts = null; private String feedId; private FileMetaGrep(String[] args) { matchMap = ArgsUtil.parse(args); - matchMap.remove("repoPath"); - matchMap.remove("feedId"); + matchMap.remove(CIKey.of("repoPath")); + matchMap.remove(CIKey.of("feedId")); doMain(args); } @@ -128,7 +129,7 @@ private void scanFile(Path file) { boolean match = true; - for (String matchKey : matchMap.keySet()) { + for (CIKey matchKey : matchMap.keySet()) { if (!attributeMap.containsKey(matchKey)) { // No Good match = false; diff --git a/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsSource.java b/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsSource.java index 1cb608a0d44..d4c82896df7 100644 --- a/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsSource.java +++ b/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsSource.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,6 +26,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKeys; import java.io.IOException; import java.io.InputStream; @@ -134,7 +135,7 @@ private void readManifest(final AttributeMap attributeMap) { try { final List files = fileSystemStreamPathHelper.getFiles(file); - attributeMap.putCollection("Files", files.stream() + attributeMap.putCollection(CIKeys.FILES, files.stream() .map(FileUtil::getCanonicalPath) .toList()); } catch (final IOException e) { diff --git a/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsStore.java b/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsStore.java index 7cfd8479969..2d5c5e58741 100644 --- a/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsStore.java +++ b/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsStore.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.data.store.impl.fs; @@ -28,9 +27,11 @@ import stroom.meta.api.MetaProperties; import stroom.meta.api.MetaService; import stroom.meta.shared.Meta; +import stroom.util.NullSafe; import stroom.util.io.PathCreator; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -193,11 +194,12 @@ public Source openSource(final long streamId, final boolean anyStatus) throws Da } @Override - public Map getAttributes(final long metaId) { + public Map getAttributes(final long metaId) { try (final Source source = openSource(metaId, true)) { - return source != null - ? source.getAttributes() - : Collections.emptyMap(); + return NullSafe.getOrElseGet( + source, + Source::getAttributes, + Collections::emptyMap); } catch (final IOException e) { throw new RuntimeException(e.getMessage(), e); } diff --git a/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsTarget.java b/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsTarget.java index 823b1e6a714..d2c56b5f425 100644 --- a/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsTarget.java +++ b/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/FsTarget.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -148,7 +148,7 @@ private void writeManifest() { private void updateAttribute(final FsTarget target, final QueryField key, final String value) { if (!target.getAttributes().containsKey(key.getFldName())) { - target.getAttributes().put(key.getFldName(), value); + target.getAttributes().put(key.getFldNameAsCIKey(), value); } } diff --git a/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/S3Target.java b/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/S3Target.java index 90f4142611e..4a8879c5212 100644 --- a/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/S3Target.java +++ b/stroom-data/stroom-data-store-impl-fs/src/main/java/stroom/data/store/impl/fs/S3Target.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -104,7 +104,7 @@ private void writeManifest() { private void updateAttribute(final S3Target target, final QueryField key, final String value) { if (!target.getAttributes().containsKey(key.getFldName())) { - target.getAttributes().put(key.getFldName(), value); + target.getAttributes().put(key.getFldNameAsCIKey(), value); } } diff --git a/stroom-data/stroom-data-store-impl-fs/src/test/java/stroom/data/store/impl/fs/ManualCheckStreamPerformance.java b/stroom-data/stroom-data-store-impl-fs/src/test/java/stroom/data/store/impl/fs/ManualCheckStreamPerformance.java index 451955f5944..0f205af8836 100644 --- a/stroom-data/stroom-data-store-impl-fs/src/test/java/stroom/data/store/impl/fs/ManualCheckStreamPerformance.java +++ b/stroom-data/stroom-data-store-impl-fs/src/test/java/stroom/data/store/impl/fs/ManualCheckStreamPerformance.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import stroom.util.io.FileUtil; import stroom.util.io.StreamUtil; import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; import com.google.common.base.Strings; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; @@ -38,6 +39,7 @@ import java.util.Map; abstract class ManualCheckStreamPerformance { + private static int testThreadCount = 10; private static int testSize = 100000; @@ -86,13 +88,13 @@ static void averageTimeCheck(final String msg, final TimedAction provider) throw } public static void main(final String[] args) throws InterruptedException { - final Map map = ArgsUtil.parse(args); + final Map map = ArgsUtil.parse(args); - if (map.containsKey("testThreadCount")) { - testThreadCount = Integer.parseInt(map.get("testThreadCount")); + if (map.containsKey(CIKey.of("testThreadCount"))) { + testThreadCount = Integer.parseInt(map.get(CIKey.of("testThreadCount"))); } - if (map.containsKey("testSize")) { - testSize = Integer.parseInt(map.get("testSize")); + if (map.containsKey(CIKey.of("testSize"))) { + testSize = Integer.parseInt(map.get(CIKey.of("testSize"))); } averageTimeCheck("W BGZIP 1000000", @@ -205,10 +207,12 @@ public long seekLargeFileTest() throws IOException { } public interface TimedAction { + long newTimedAction() throws IOException; } public static class BlockGzipManualCheckStreamPerformance extends ManualCheckStreamPerformance { + Path tempFile; int blockSize; long blockCount; @@ -245,6 +249,7 @@ public String toString() { } public static class UncompressedCheckStreamPerformance extends ManualCheckStreamPerformance { + Path tempFile; @Override @@ -274,6 +279,7 @@ public void onCloseOutput(final OutputStream arg0) { } public static class GzipCheckStreamPerformance extends ManualCheckStreamPerformance { + Path tempFile; @Override diff --git a/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/AttributeMapFactory.java b/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/AttributeMapFactory.java index cf61e87ba5e..452cf8524b8 100644 --- a/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/AttributeMapFactory.java +++ b/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/AttributeMapFactory.java @@ -1,10 +1,26 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.data.store.impl; -import stroom.meta.shared.Meta; +import stroom.util.shared.string.CIKey; import java.util.Map; public interface AttributeMapFactory { - Map getAttributes(long metaId); + Map getAttributes(long metaId); } diff --git a/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/DataResourceImpl.java b/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/DataResourceImpl.java index 746201dc0e9..554c8d3ad07 100644 --- a/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/DataResourceImpl.java +++ b/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/DataResourceImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,6 +36,7 @@ import stroom.util.shared.OffsetRange; import stroom.util.shared.ResourceGeneration; import stroom.util.shared.ResourceKey; +import stroom.util.shared.string.CIKey; import event.logging.ComplexLoggedOutcome; import event.logging.ExportEventAction; @@ -118,9 +119,9 @@ public ResourceKey upload(final UploadDataRequest request) { @Override public Map metaAttributes(final long id) { - final Map result; try { - return dataServiceProvider.get().metaAttributes(id); + final Map map = dataServiceProvider.get().metaAttributes(id); + return CIKey.convertToStringMap(map); } catch (final RuntimeException e) { throw EntityServiceExceptionUtil.create(e); } diff --git a/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/DataServiceImpl.java b/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/DataServiceImpl.java index 62ffb9a69f2..f6c46ba9a1a 100644 --- a/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/DataServiceImpl.java +++ b/stroom-data/stroom-data-store-impl/src/main/java/stroom/data/store/impl/DataServiceImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -65,6 +65,8 @@ import stroom.util.shared.ResourceGeneration; import stroom.util.shared.ResourceKey; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -82,6 +84,8 @@ class DataServiceImpl implements DataService { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(DataServiceImpl.class); + private static final CIKey FILES_ATTR_KEY = CIKeys.FILES; + private final ResourceStore resourceStore; private final DataUploadTaskHandler dataUploadTaskHandlerProvider; private final DataDownloadTaskHandler dataDownloadTaskHandlerProvider; @@ -211,7 +215,7 @@ public ResourceKey upload(final UploadDataRequest request) { } @Override - public Map metaAttributes(final long id) { + public Map metaAttributes(final long id) { return attributeMapFactory.getAttributes(id); } @@ -232,37 +236,38 @@ public List info(final long id) { final List entries = new ArrayList<>(); final Map attributeMap = metaRow.getAttributes(); - final Map additionalAttributes = attributeMapFactory.getAttributes( + final Map additionalAttributes = attributeMapFactory.getAttributes( metaRow.getMeta().getId()); - final String files = additionalAttributes.remove("Files"); - attributeMap.putAll(additionalAttributes); + final String files = additionalAttributes.remove(FILES_ATTR_KEY); + additionalAttributes + .forEach((k, v) -> + attributeMap.put(k.get(), v)); - final List sortedKeys = attributeMap - .keySet() + attributeMap.entrySet() .stream() - .sorted() - .toList(); - sortedKeys.forEach(key -> { - final String value = attributeMap.get(key); - if (value != null && - // We are going to add retention entries separately. - !DataRetentionFields.RETENTION_AGE.equals(key) && - !DataRetentionFields.RETENTION_UNTIL.equals(key) && - !DataRetentionFields.RETENTION_RULE.equals(key)) { - - if (MetaFields.DURATION.getFldName().equals(key)) { - entries.add(new DataInfoSection.Entry(key, convertDuration(value))); - } else if (key.toLowerCase().contains("time")) { - entries.add(new DataInfoSection.Entry(key, convertTime(value))); - } else if (key.toLowerCase().contains("size")) { - entries.add(new DataInfoSection.Entry(key, convertSize(value))); - } else if (key.toLowerCase().contains("count")) { - entries.add(new DataInfoSection.Entry(key, convertCount(value))); - } else { - entries.add(new DataInfoSection.Entry(key, value)); - } - } - }); + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> { + final String key = entry.getKey(); + final String value = entry.getValue(); + if (value != null && + // We are going to add retention entries separately. + !DataRetentionFields.RETENTION_AGE_FIELD.getFldName().equals(key) && + !DataRetentionFields.RETENTION_UNTIL_FIELD.getFldName().equals(key) && + !DataRetentionFields.RETENTION_RULE_FIELD.getFldName().equals(key)) { + + if (MetaFields.DURATION.getFldName().equalsIgnoreCase(key)) { + entries.add(new DataInfoSection.Entry(key, convertDuration(value))); + } else if (key.toLowerCase().contains("time")) { + entries.add(new DataInfoSection.Entry(key, convertTime(value))); + } else if (key.toLowerCase().contains("size")) { + entries.add(new DataInfoSection.Entry(key, convertSize(value))); + } else if (key.toLowerCase().contains("count")) { + entries.add(new DataInfoSection.Entry(key, convertCount(value))); + } else { + entries.add(new DataInfoSection.Entry(key, value)); + } + } + }); sections.add(new DataInfoSection("Attributes", entries)); // Add additional data retention information. @@ -426,13 +431,13 @@ private String getDateTimeString(final Long ms) { private List getDataRententionEntries(final Map attributeMap) { final List entries = new ArrayList<>(); - if (attributeMap != null && !attributeMap.isEmpty()) { + if (NullSafe.hasEntries(attributeMap)) { entries.add(new DataInfoSection.Entry(DataRetentionFields.RETENTION_AGE, - attributeMap.get(DataRetentionFields.RETENTION_AGE))); + attributeMap.get(DataRetentionFields.RETENTION_AGE_FIELD.getFldName()))); entries.add(new DataInfoSection.Entry(DataRetentionFields.RETENTION_UNTIL, - attributeMap.get(DataRetentionFields.RETENTION_UNTIL))); + attributeMap.get(DataRetentionFields.RETENTION_UNTIL_FIELD.getFldName()))); entries.add(new DataInfoSection.Entry(DataRetentionFields.RETENTION_RULE, - attributeMap.get(DataRetentionFields.RETENTION_RULE))); + attributeMap.get(DataRetentionFields.RETENTION_RULE_FIELD.getFldName()))); } return entries; diff --git a/stroom-data/stroom-data-store-mock/src/main/java/stroom/data/store/mock/MockDataService.java b/stroom-data/stroom-data-store-mock/src/main/java/stroom/data/store/mock/MockDataService.java index 7fe6c46df9a..1ab7daac1de 100644 --- a/stroom-data/stroom-data-store-mock/src/main/java/stroom/data/store/mock/MockDataService.java +++ b/stroom-data/stroom-data-store-mock/src/main/java/stroom/data/store/mock/MockDataService.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.data.store.mock; import stroom.data.shared.DataInfoSection; @@ -8,6 +24,7 @@ import stroom.pipeline.shared.FetchDataRequest; import stroom.util.shared.ResourceGeneration; import stroom.util.shared.ResourceKey; +import stroom.util.shared.string.CIKey; import java.util.List; import java.util.Map; @@ -26,7 +43,7 @@ public ResourceKey upload(final UploadDataRequest request) { } @Override - public Map metaAttributes(final long id) { + public Map metaAttributes(final long id) { return Map.of(); } diff --git a/stroom-data/stroom-data-zip/src/main/java/stroom/data/zip/PathCreator.java b/stroom-data/stroom-data-zip/src/main/java/stroom/data/zip/PathCreator.java index 7c286f4f604..371b97840a7 100644 --- a/stroom-data/stroom-data-zip/src/main/java/stroom/data/zip/PathCreator.java +++ b/stroom-data/stroom-data-zip/src/main/java/stroom/data/zip/PathCreator.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.data.zip; import stroom.meta.api.AttributeMap; diff --git a/stroom-db-util/src/main/java/stroom/db/util/CommonExpressionMapper.java b/stroom-db-util/src/main/java/stroom/db/util/CommonExpressionMapper.java index d82750c0800..8137e6f4718 100644 --- a/stroom-db-util/src/main/java/stroom/db/util/CommonExpressionMapper.java +++ b/stroom-db-util/src/main/java/stroom/db/util/CommonExpressionMapper.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.db.util; import stroom.datasource.api.v2.FieldType; @@ -9,6 +25,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; import org.jooq.Condition; import org.jooq.impl.DSL; @@ -28,9 +45,9 @@ public final class CommonExpressionMapper implements Function> termHandlers = new HashMap<>(); - private final Map fieldMap = new HashMap<>(); - private final Set ignoredFields = new HashSet<>(); + private final Map> termHandlers = new HashMap<>(); + private final Map fieldMap = new HashMap<>(); + private final Set ignoredFields = new HashSet<>(); private final Function delegateItemHandler; public CommonExpressionMapper() { @@ -43,17 +60,17 @@ public CommonExpressionMapper(final Function delegate public void addHandler(final QueryField dataSourceField, final Function handler) { - final String fieldName = dataSourceField.getFldName(); + final CIKey fieldName = dataSourceField.getFldNameAsCIKey(); termHandlers.put(fieldName, handler); fieldMap.put(fieldName, dataSourceField); } public void ignoreField(final QueryField dataSourceField) { - ignoredFields.add(dataSourceField.getFldName()); + ignoredFields.add(dataSourceField.getFldNameAsCIKey()); } /** - * Converts the passed {@link ExpressionItem} into a Jooq {@link Condition}. By default it + * Converts the passed {@link ExpressionItem} into a Jooq {@link Condition}. By default, it * will simplify expressions that can be simplified, e.g. NOT {NOT{}} becomes true, an OR * with one child that is true becomes true, etc. It will always return a value. */ @@ -72,8 +89,8 @@ public Optional innerApply(final ExpressionItem item) { if (item != null && item.enabled()) { if (item instanceof final ExpressionTerm term) { - final String fieldName = term.getField(); - if (fieldName == null) { + final CIKey fieldName = CIKey.of(term.getField()); + if (CIKey.isNull(fieldName)) { throw new NullPointerException("Term has a null field '" + term + "'"); } if (term.getCondition() == null) { @@ -103,7 +120,7 @@ public Optional innerApply(final ExpressionItem item) { result = Optional.of(termHandler.apply(term)); } else if (delegateItemHandler != null) { result = Optional.of(delegateItemHandler.apply(term)); - } else if (!ignoredFields.contains(term.getField())) { + } else if (!ignoredFields.contains(fieldName)) { throw new RuntimeException("No term handler supplied for term '" + term.getField() + "'"); } diff --git a/stroom-db-util/src/main/java/stroom/db/util/JooqUtil.java b/stroom-db-util/src/main/java/stroom/db/util/JooqUtil.java index 1bdaad974de..9ef76fdb099 100644 --- a/stroom-db-util/src/main/java/stroom/db/util/JooqUtil.java +++ b/stroom-db-util/src/main/java/stroom/db/util/JooqUtil.java @@ -34,8 +34,10 @@ import stroom.util.shared.Range; import stroom.util.shared.Selection; import stroom.util.shared.StringCriteria; +import stroom.util.shared.string.CIKey; import stroom.util.string.PatternUtil; +import org.jooq.Collation; import org.jooq.Condition; import org.jooq.Configuration; import org.jooq.DSLContext; @@ -88,6 +90,12 @@ public final class JooqUtil { static final int MAX_DEADLOCK_RETRY_ATTEMPTS = 20; private static final long SLEEP_INCREMENT_MS = 10; + /** + * The collation to use if you want case sensitivity. By default, our tables use {@code utf8mb4_0900_ai_ci}. + */ + public static final String CASE_SENSITIVE_COLLATION_NAME = "utf8mb4_0900_as_cs"; + public static final Collation CASE_SENSITIVE_COLLATION = DSL.collation(CASE_SENSITIVE_COLLATION_NAME); + private JooqUtil() { // Utility class. } @@ -96,6 +104,11 @@ public static void disableJooqLogoInLogs() { System.getProperties().setProperty("org.jooq.no-logo", "true"); } + public static Field withCaseSensitiveCollation(final Field field) { + return NullSafe.get(field, + field2 -> field2.collate(CASE_SENSITIVE_COLLATION_NAME)); + } + private static Settings createSettings(final boolean isExecuteWithOptimisticLocking) { Settings settings = new Settings(); // Turn off fully qualified schemata. @@ -886,7 +899,7 @@ private static Optional convertMatchNull(final Field field, return condition.or(() -> Optional.of(field.isNotNull())); } - public static Collection> getOrderFields(final Map> fieldMap, + public static Collection> getOrderFields(final Map> fieldMap, final BaseCriteria criteria, final OrderField... defaultSortFields) { if (criteria.getSortList() == null || criteria.getSortList().isEmpty()) { @@ -905,9 +918,9 @@ public static Collection> getOrderFields(final Map> getOrderField(final Map> fieldMap, + private static Optional> getOrderField(final Map> fieldMap, final CriteriaFieldSort sort) { - final Field field = fieldMap.get(sort.getId()); + final Field field = fieldMap.get(CIKey.of(sort.getId())); if (field != null) { if (sort.isDesc()) { diff --git a/stroom-db-util/src/main/java/stroom/db/util/StringMatchConditionUtil.java b/stroom-db-util/src/main/java/stroom/db/util/StringMatchConditionUtil.java index dca7e31e03e..d32af975c0e 100644 --- a/stroom-db-util/src/main/java/stroom/db/util/StringMatchConditionUtil.java +++ b/stroom-db-util/src/main/java/stroom/db/util/StringMatchConditionUtil.java @@ -17,6 +17,8 @@ package stroom.db.util; import stroom.docref.StringMatch; +import stroom.util.logging.LambdaLogger; +import stroom.util.logging.LambdaLoggerFactory; import org.jooq.Condition; import org.jooq.Field; @@ -24,68 +26,112 @@ import java.util.stream.Collectors; +// This is effectively tested via IndexFieldDaoImpl public class StringMatchConditionUtil { + private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(StringMatchConditionUtil.class); + private StringMatchConditionUtil() { // Util. } public static Condition getCondition(final Field field, final StringMatch stringMatch) { - Condition condition = DSL.trueCondition(); - if (stringMatch != null) { - switch (stringMatch.getMatchType()) { - case ANY -> condition = DSL.trueCondition(); - case NULL -> condition = field.isNull(); - case NON_NULL -> condition = field.isNotNull(); - case BLANK -> condition = field.isNotNull().and(DSL.trim(field).equal("")); - case NON_BLANK -> condition = field.isNotNull().and(DSL.trim(field).notEqual("")); - case EMPTY -> condition = field.eq(""); - case NON_EMPTY -> condition = field.isNotNull().and(field.notEqual("")); - case NULL_OR_BLANK -> condition = field.isNull().or(DSL.trim(field).equal("")); - case NULL_OR_EMPTY -> condition = field.isNull().or(field.eq("")); - case CONTAINS -> { - if (stringMatch.isCaseSensitive()) { - condition = field.contains(stringMatch.getPattern()); - } else { - condition = field.containsIgnoreCase(stringMatch.getPattern()); - } - } - case EQUALS -> { - if (stringMatch.isCaseSensitive()) { - condition = field.equal(stringMatch.getPattern()); - } else { - condition = field.equalIgnoreCase(stringMatch.getPattern()); - } - } - case NOT_EQUALS -> { - if (stringMatch.isCaseSensitive()) { - condition = field.notEqual(stringMatch.getPattern()); - } else { - condition = field.notEqualIgnoreCase(stringMatch.getPattern()); - } - } - case STARTS_WITH -> { - if (stringMatch.isCaseSensitive()) { - condition = field.startsWith(stringMatch.getPattern()); - } else { - condition = field.startsWithIgnoreCase(stringMatch.getPattern()); - } - } - case ENDS_WITH -> { - if (stringMatch.isCaseSensitive()) { - condition = field.endsWith(stringMatch.getPattern()); - } else { - condition = field.endsWithIgnoreCase(stringMatch.getPattern()); - } - } - case REGEX -> condition = field.likeRegex(stringMatch.getPattern()); - case CHARS_ANYWHERE -> condition = getCharsAnywhereLikePattern(field, stringMatch); - } + final Condition condition; + if (stringMatch == null) { + condition = DSL.trueCondition(); + } else { + condition = switch (stringMatch.getMatchType()) { + case NULL -> field.isNull(); + case NON_NULL -> field.isNotNull(); + case BLANK -> field.isNotNull().and(DSL.trim(field).equal("")); + case NON_BLANK -> field.isNotNull().and(DSL.trim(field).notEqual("")); + case EMPTY -> field.eq(""); + case NON_EMPTY -> field.isNotNull().and(field.notEqual("")); + case NULL_OR_BLANK -> field.isNull().or(DSL.trim(field).equal("")); + case NULL_OR_EMPTY -> field.isNull().or(field.eq("")); + + // IMPORTANT: + // At the time of writing, the collation on all the tables is utf8mb4_0900_ai_ci, + // which is insensitive on accent and case. If there is a unique key on a varchar + // column, and you try to insert 'foo' and 'FOO' the latter will violate the key due + // to the collation. Thus, even if we query in a case-sensitive way, the data may already + // be affected by unique constraints in place. + + case CONTAINS -> getContainsCondition(field, stringMatch); + case EQUALS -> getEqualsCondition(field, stringMatch); + case NOT_EQUALS -> getNotEqualsCondition(field, stringMatch); + case STARTS_WITH -> getStartsWithCondition(field, stringMatch); + case ENDS_WITH -> getEndsWithCondition(field, stringMatch); + case REGEX -> getRegexCondition(field, stringMatch); + case CHARS_ANYWHERE -> getCharsAnywhereLikePattern(field, stringMatch); + default -> DSL.trueCondition(); + }; } + LOGGER.trace("field: {}, stringMatch: {}, condition: {}", field, stringMatch, condition); return condition; } + private static Condition getEndsWithCondition(final Field field, final StringMatch stringMatch) { + if (stringMatch.isCaseSensitive()) { + // See note above + return JooqUtil.withCaseSensitiveCollation(field) + .endsWith(stringMatch.getPattern()); + } else { + return field.endsWithIgnoreCase(stringMatch.getPattern()); + } + } + + private static Condition getStartsWithCondition(final Field field, final StringMatch stringMatch) { + if (stringMatch.isCaseSensitive()) { + // See note above + return JooqUtil.withCaseSensitiveCollation(field) + .startsWith(stringMatch.getPattern()); + } else { + return field.startsWithIgnoreCase(stringMatch.getPattern()); + } + } + + private static Condition getNotEqualsCondition(final Field field, final StringMatch stringMatch) { + if (stringMatch.isCaseSensitive()) { + // See note above + return JooqUtil.withCaseSensitiveCollation(field) + .notEqual(stringMatch.getPattern()); + } else { + return field.notEqualIgnoreCase(stringMatch.getPattern()); + } + } + + private static Condition getEqualsCondition(final Field field, final StringMatch stringMatch) { + if (stringMatch.isCaseSensitive()) { + // See note above + return JooqUtil.withCaseSensitiveCollation(field) + .equal(stringMatch.getPattern()); + } else { + return field.equalIgnoreCase(stringMatch.getPattern()); + } + } + + private static Condition getContainsCondition(final Field field, final StringMatch stringMatch) { + if (stringMatch.isCaseSensitive()) { + // See note above + return JooqUtil.withCaseSensitiveCollation(field) + .contains(stringMatch.getPattern()); + } else { + return field.containsIgnoreCase(stringMatch.getPattern()); + } + } + + private static Condition getRegexCondition(final Field field, final StringMatch stringMatch) { + if (stringMatch.isCaseSensitive()) { + // See note above + return JooqUtil.withCaseSensitiveCollation(field) + .likeRegex(stringMatch.getPattern()); + } else { + return field.likeRegex(stringMatch.getPattern()); + } + } + private static Condition getCharsAnywhereLikePattern(final Field field, final StringMatch stringMatch) { String effectivePattern = stringMatch.getPattern().chars() diff --git a/stroom-db-util/src/main/java/stroom/db/util/ValueMapper.java b/stroom-db-util/src/main/java/stroom/db/util/ValueMapper.java index 4326c5e9f35..b20c2cb9ce8 100644 --- a/stroom-db-util/src/main/java/stroom/db/util/ValueMapper.java +++ b/stroom-db-util/src/main/java/stroom/db/util/ValueMapper.java @@ -1,13 +1,30 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.db.util; import stroom.datasource.api.v2.QueryField; import stroom.query.language.functions.Val; import stroom.query.language.functions.ValNull; +import stroom.util.shared.GwtNullSafe; +import stroom.util.shared.string.CIKey; import org.jooq.Field; import org.jooq.Record; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -17,30 +34,34 @@ public class ValueMapper { - private final Map> fieldNameMap = new HashMap<>(); - private final Map> mappers = new HashMap<>(); + private final Map> fieldNameMap = new HashMap<>(); + private final Map> mappers = new HashMap<>(); - public void map(final QueryField dataSourceField, final Field field, final Function handler) { - fieldNameMap.put(dataSourceField.getFldName(), field); - mappers.put(dataSourceField.getFldName(), new Mapper<>(field, handler)); + public void map(final QueryField dataSourceField, + final Field field, + final Function handler) { + final CIKey fldNameAsCIKey = dataSourceField.getFldNameAsCIKey(); + fieldNameMap.put(fldNameAsCIKey, field); + mappers.put(fldNameAsCIKey, new Mapper<>(field, handler)); } - public List> getDbFieldsByName(final String[] fieldNames) { - return Arrays - .stream(fieldNames) + public List> getDbFieldsByName(final List fieldNames) { + return GwtNullSafe.stream(fieldNames) .map(fieldNameMap::get) .filter(Objects::nonNull) .collect(Collectors.toList()); } - public Mapper[] getMappersForFieldNames(final String[] fieldNames) { - final Mapper[] handlers = new Mapper[fieldNames.length]; - for (int i = 0; i < fieldNames.length; i++) { - handlers[i] = mappers.get(fieldNames[i]); - } - return handlers; + public Mapper[] getMappersForFieldNames(final List fieldNames) { + return GwtNullSafe.stream(fieldNames) + .map(mappers::get) + .toArray(Mapper[]::new); } + + // -------------------------------------------------------------------------------- + + public static class Mapper { private final Field field; diff --git a/stroom-dictionary/stroom-dictionary-api/src/main/java/stroom/dictionary/api/WordListProvider.java b/stroom-dictionary/stroom-dictionary-api/src/main/java/stroom/dictionary/api/WordListProvider.java index e30644f6b49..64e1dca3fbf 100644 --- a/stroom-dictionary/stroom-dictionary-api/src/main/java/stroom/dictionary/api/WordListProvider.java +++ b/stroom-dictionary/stroom-dictionary-api/src/main/java/stroom/dictionary/api/WordListProvider.java @@ -18,12 +18,12 @@ import stroom.dictionary.shared.WordList; import stroom.docref.DocRef; +import stroom.docref.HasFindDocsByName; import stroom.docrefinfo.api.DocRefDecorator; -import java.util.List; import java.util.Optional; -public interface WordListProvider { +public interface WordListProvider extends HasFindDocsByName { /** * @return The complete wordList as a single string with words delimited by {@code \n}. @@ -51,7 +51,5 @@ public interface WordListProvider { */ WordList getCombinedWordList(DocRef dictionaryRef, DocRefDecorator docRefDecorator); - List findByName(String name); - Optional findByUuid(String uuid); } diff --git a/stroom-dictionary/stroom-dictionary-impl/src/main/java/stroom/dictionary/impl/DictionaryStoreImpl.java b/stroom-dictionary/stroom-dictionary-impl/src/main/java/stroom/dictionary/impl/DictionaryStoreImpl.java index 8122d8d69c6..8b1369e1189 100644 --- a/stroom-dictionary/stroom-dictionary-impl/src/main/java/stroom/dictionary/impl/DictionaryStoreImpl.java +++ b/stroom-dictionary/stroom-dictionary-impl/src/main/java/stroom/dictionary/impl/DictionaryStoreImpl.java @@ -268,13 +268,10 @@ public Optional findByUuid(final String uuid) { } @Override - public List findByName(final String name) { - return findByNames(List.of(name), false); - } - - @Override - public List findByNames(final List names, final boolean allowWildCards) { - return store.findByNames(names, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-dictionary/stroom-dictionary-mock/src/main/java/stroom/dictionary/mock/MockWordListProviderModule.java b/stroom-dictionary/stroom-dictionary-mock/src/main/java/stroom/dictionary/mock/MockWordListProviderModule.java index e7ce2ac1d3a..e4d9ccb0d5c 100644 --- a/stroom-dictionary/stroom-dictionary-mock/src/main/java/stroom/dictionary/mock/MockWordListProviderModule.java +++ b/stroom-dictionary/stroom-dictionary-mock/src/main/java/stroom/dictionary/mock/MockWordListProviderModule.java @@ -24,8 +24,10 @@ import com.google.inject.AbstractModule; import com.google.inject.Provides; +import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; public class MockWordListProviderModule extends AbstractModule { @@ -34,8 +36,8 @@ WordListProvider wordListProvider() { return new WordListProvider() { @Override - public List findByName(final String name) { - return List.of(); + public Set listDocuments() { + return Collections.emptySet(); } @Override @@ -43,6 +45,20 @@ public Optional findByUuid(final String uuid) { return Optional.empty(); } + @Override + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return Collections.emptyList(); + } + + // @Override +// public List findByNames(final List names, +// final boolean allowWildCards, +// final boolean isCaseSensitive) { +// return Collections.emptyList(); +// } + @Override public String getCombinedData(final DocRef dictionaryRef) { return null; diff --git a/stroom-docref/src/main/java/stroom/docref/DocRef.java b/stroom-docref/src/main/java/stroom/docref/DocRef.java index ad904f698c3..e63426f2503 100644 --- a/stroom-docref/src/main/java/stroom/docref/DocRef.java +++ b/stroom-docref/src/main/java/stroom/docref/DocRef.java @@ -64,8 +64,9 @@ public final class DocRef implements Comparable, HasDisplayValue, HasTyp private String name; /** - * JAXB requires a no-arg constructor. + * Only here for XML ser/deser. Don't use! */ + @Deprecated public DocRef() { } @@ -102,10 +103,15 @@ public String getType() { return type; } + /** + * Only here for XML ser/deser. Don't use! + */ + @Deprecated public void setType(final String type) { this.type = type; } + /** * @return A UUID as generated by {@link java.util.UUID#randomUUID()} */ @@ -114,6 +120,10 @@ public String getUuid() { return uuid; } + /** + * Only here for XML ser/deser. Don't use! + */ + @Deprecated public void setUuid(final String uuid) { this.uuid = uuid; } @@ -126,6 +136,10 @@ public String getName() { return name; } + /** + * Only here for XML ser/deser. Don't use! + */ + @Deprecated @Override public void setName(final String name) { this.name = name; diff --git a/stroom-docref/src/main/java/stroom/docref/HasFindDocsByName.java b/stroom-docref/src/main/java/stroom/docref/HasFindDocsByName.java index 7eb1037b305..af3ea659be1 100644 --- a/stroom-docref/src/main/java/stroom/docref/HasFindDocsByName.java +++ b/stroom-docref/src/main/java/stroom/docref/HasFindDocsByName.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.docref; import java.util.Collections; @@ -17,7 +33,7 @@ public interface HasFindDocsByName { default List findByName(final String name) { // GWT so no List.of() return name != null - ? findByNames(Collections.singletonList(name), false) + ? findByNames(Collections.singletonList(name), false, true) : Collections.emptyList(); } @@ -25,10 +41,12 @@ default List findByName(final String name) { * Find by case-sensitive match on the name. * If allowWildCards is true '*' can be used to denote a 0-many char wild card. */ - default List findByName(final String name, final boolean allowWildCards) { + default List findByName(final String name, + final boolean allowWildCards, + final boolean isCaseSensitive) { // GWT so no List.of() return name != null - ? findByNames(Collections.singletonList(name), allowWildCards) + ? findByNames(Collections.singletonList(name), allowWildCards, isCaseSensitive) : Collections.emptyList(); } @@ -37,5 +55,7 @@ default List findByName(final String name, final boolean allowWildCards) * If allowWildCards is true '*' can be used to denote a 0-many char wild card. * Finds all docRefs associated with any of the names, i.e. an OR. */ - List findByNames(List names, boolean allowWildCards); + List findByNames(List names, + boolean allowWildCards, + final boolean isCaseSensitive); } diff --git a/stroom-docref/src/main/java/stroom/docref/StringMatch.java b/stroom-docref/src/main/java/stroom/docref/StringMatch.java index 3594952520c..114d88fde6e 100644 --- a/stroom-docref/src/main/java/stroom/docref/StringMatch.java +++ b/stroom-docref/src/main/java/stroom/docref/StringMatch.java @@ -24,6 +24,14 @@ import java.util.Objects; +/** + * Be aware that the default collation of the database tables and columns is {@code utf8mb4_0900_ai_ci} + * which is insensitive on case and accent, which would prevent 'foo' and 'FOO' from existing if there + * is a unique key in place on that column. + *

+ * Thus, you should bear this limitation in mind if you are wanting to match in a case-sensitive way. + *

+ */ @JsonPropertyOrder({"matchType", "caseSensitive", "pattern"}) @JsonInclude(Include.NON_NULL) public class StringMatch { @@ -55,7 +63,17 @@ public static StringMatch nullOrBlank() { return new StringMatch(MatchType.NULL_OR_BLANK, false, null); } + /** + * Case-sensitive contains match + */ public static StringMatch contains(final String pattern) { + return contains(pattern, true); + } + + /** + * Case-insensitive contains match + */ + public static StringMatch containsIgnoreCase(final String pattern) { return contains(pattern, false); } @@ -66,7 +84,17 @@ public static StringMatch contains(final String pattern, final boolean caseSensi return new StringMatch(MatchType.CONTAINS, caseSensitive, pattern); } + /** + * Case-sensitive equals match + */ public static StringMatch equals(final String pattern) { + return equals(pattern, true); + } + + /** + * Case-insensitive equals match + */ + public static StringMatch equalsIgnoreCase(final String pattern) { return equals(pattern, false); } @@ -74,7 +102,17 @@ public static StringMatch equals(final String pattern, final boolean caseSensiti return new StringMatch(MatchType.EQUALS, caseSensitive, pattern); } + /** + * Case-sensitive NOT equals match + */ public static StringMatch notEquals(final String pattern) { + return notEquals(pattern, true); + } + + /** + * Case-insensitive NOT equals match + */ + public static StringMatch notEqualsIgnoreCase(final String pattern) { return notEquals(pattern, false); } @@ -82,7 +120,17 @@ public static StringMatch notEquals(final String pattern, final boolean caseSens return new StringMatch(MatchType.NOT_EQUALS, caseSensitive, pattern); } + /** + * Case-sensitive regex match + */ public static StringMatch regex(final String pattern) { + return regex(pattern, true); + } + + /** + * Case-insensitive regex match + */ + public static StringMatch regexIgnoreCase(final String pattern) { return regex(pattern, false); } @@ -119,6 +167,7 @@ public boolean equals(final Object o) { if (!(o instanceof StringMatch)) { return false; } + @SuppressWarnings("PatternVariableCanBeUsed") // GWT final StringMatch stringMatch = (StringMatch) o; return caseSensitive == stringMatch.caseSensitive && matchType == stringMatch.matchType && @@ -130,6 +179,18 @@ public int hashCode() { return Objects.hash(matchType, caseSensitive, pattern); } + @Override + public String toString() { + return matchType + + ( + pattern == null + ? "" + : " '" + pattern + "'") + + " (" + ( + caseSensitive + ? "case-sensitive)" + : "case-insensitive)"); + } // -------------------------------------------------------------------------------- diff --git a/stroom-docref/src/test/java/stroom/docref/TestStringMatch.java b/stroom-docref/src/test/java/stroom/docref/TestStringMatch.java new file mode 100644 index 00000000000..80ec2be0dff --- /dev/null +++ b/stroom-docref/src/test/java/stroom/docref/TestStringMatch.java @@ -0,0 +1,47 @@ +package stroom.docref; + +import stroom.test.common.TestUtil; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; + +import java.util.stream.Stream; + +class TestStringMatch { + + @TestFactory + Stream testIsCaseSensitive() { + final String pattern = "foo"; + + return TestUtil.buildDynamicTestStream() + .withInputType(StringMatch.class) + .withOutputType(boolean.class) + .withSingleArgTestFunction(StringMatch::isCaseSensitive) + .withAssertions(outcome -> { + final StringMatch stringMatch = outcome.getInput(); + Assertions.assertThat(outcome.getActualOutput()) + .isEqualTo(outcome.getExpectedOutput()); + Assertions.assertThat(stringMatch.getPattern()) + .isEqualTo(pattern); + }) + + .addCase(StringMatch.equals(pattern), true) + .addCase(StringMatch.equals(pattern, false), false) + .addCase(StringMatch.equalsIgnoreCase(pattern), false) + + .addCase(StringMatch.notEquals(pattern), true) + .addCase(StringMatch.notEquals(pattern, false), false) + .addCase(StringMatch.notEqualsIgnoreCase(pattern), false) + + .addCase(StringMatch.contains(pattern), true) + .addCase(StringMatch.contains(pattern, false), false) + .addCase(StringMatch.containsIgnoreCase(pattern), false) + + .addCase(StringMatch.regex(pattern), true) + .addCase(StringMatch.regex(pattern, false), false) + .addCase(StringMatch.regexIgnoreCase(pattern), false) + + .build(); + } +} diff --git a/stroom-docstore/stroom-docstore-impl-db/src/main/java/stroom/docstore/impl/db/DBPersistence.java b/stroom-docstore/stroom-docstore-impl-db/src/main/java/stroom/docstore/impl/db/DBPersistence.java index 9079a2601b6..372f4f94886 100644 --- a/stroom-docstore/stroom-docstore-impl-db/src/main/java/stroom/docstore/impl/db/DBPersistence.java +++ b/stroom-docstore/stroom-docstore-impl-db/src/main/java/stroom/docstore/impl/db/DBPersistence.java @@ -16,10 +16,12 @@ package stroom.docstore.impl.db; +import stroom.db.util.JooqUtil; import stroom.docref.DocRef; import stroom.docstore.api.DocumentNotFoundException; import stroom.docstore.api.RWLockFactory; import stroom.docstore.impl.Persistence; +import stroom.util.logging.LogUtil; import stroom.util.string.PatternUtil; import jakarta.inject.Inject; @@ -65,22 +67,13 @@ public class DBPersistence implements Persistence { WHERE type = ? ORDER BY uuid"""; - private static final String SELECT_BY_TYPE_NAME_EQUALS_SQL = """ + private static final String SELECT_BY_TYPE_NAME_SQL = """ SELECT DISTINCT uuid, name FROM doc WHERE type = ? - AND name = ? - ORDER BY uuid"""; - - private static final String SELECT_BY_TYPE_NAME_WILDCARD_SQL = """ - SELECT DISTINCT - uuid, - name - FROM doc - WHERE type = ? - AND name like ? + AND name {} ? ORDER BY uuid"""; private static final String SELECT_ID_BY_TYPE_UUID_SQL = """ @@ -255,15 +248,21 @@ public List list(final String type) { @Override public List find(final String type, final String nameFilter, - final boolean allowWildCards) { + final boolean allowWildCards, + final boolean isCaseSensitive) { final List list = new ArrayList<>(); final String nameFilterSqlValue = allowWildCards ? PatternUtil.createSqlLikeStringFromWildCardFilter(nameFilter) : nameFilter; - final String sql = allowWildCards - ? SELECT_BY_TYPE_NAME_WILDCARD_SQL - : SELECT_BY_TYPE_NAME_EQUALS_SQL; + // By default, the collation in mysql is case-insensitive + String condition = isCaseSensitive + ? "collate " + JooqUtil.CASE_SENSITIVE_COLLATION_NAME + " " + : ""; + condition = allowWildCards + ? condition + "like" + : condition + "="; + final String sql = LogUtil.message(SELECT_BY_TYPE_NAME_SQL, condition); try (final Connection connection = dataSource.getConnection()) { try (final PreparedStatement preparedStatement = connection.prepareStatement(sql)) { diff --git a/stroom-docstore/stroom-docstore-impl/src/main/java/stroom/docstore/impl/Persistence.java b/stroom-docstore/stroom-docstore-impl/src/main/java/stroom/docstore/impl/Persistence.java index 59bf25d2d52..67891abb07e 100644 --- a/stroom-docstore/stroom-docstore-impl/src/main/java/stroom/docstore/impl/Persistence.java +++ b/stroom-docstore/stroom-docstore-impl/src/main/java/stroom/docstore/impl/Persistence.java @@ -33,11 +33,12 @@ public interface Persistence { */ default List find(final String type, final String nameFilter, - final boolean allowWildCards) { + final boolean allowWildCards, + final boolean isCaseSensitive) { // Default impl that does all filtering in java. Not efficient for DB impls. return nameFilter == null ? Collections.emptyList() - : find(type, List.of(nameFilter), allowWildCards); + : find(type, List.of(nameFilter), allowWildCards, isCaseSensitive); } /** @@ -46,7 +47,8 @@ default List find(final String type, */ default List find(final String type, final List nameFilters, - final boolean allowWildCards) { + final boolean allowWildCards, + final boolean isCaseSensitive) { // Default impl that does all filtering in java. Not efficient for DB impls. if (NullSafe.isEmptyCollection(nameFilters)) { return Collections.emptyList(); @@ -57,12 +59,15 @@ default List find(final String type, final Predicate predicate; if (allowWildCards && PatternUtil.containsWildCards(nameFilter)) { final Pattern pattern = PatternUtil.createPatternFromWildCardFilter( - nameFilter, true); + nameFilter, true, isCaseSensitive); predicate = docRef -> pattern.matcher(docRef.getName()).matches(); - } else { + } else if (isCaseSensitive) { predicate = docRef -> nameFilter.equals(docRef.getName()); + } else { + predicate = docRef -> + nameFilter.equalsIgnoreCase(docRef.getName()); } return predicate; }) diff --git a/stroom-docstore/stroom-docstore-impl/src/main/java/stroom/docstore/impl/StoreImpl.java b/stroom-docstore/stroom-docstore-impl/src/main/java/stroom/docstore/impl/StoreImpl.java index 2ba99bedc22..e9751e1b234 100644 --- a/stroom-docstore/stroom-docstore-impl/src/main/java/stroom/docstore/impl/StoreImpl.java +++ b/stroom-docstore/stroom-docstore-impl/src/main/java/stroom/docstore/impl/StoreImpl.java @@ -677,8 +677,10 @@ public List list() { } @Override - public List findByNames(final List names, final boolean allowWildCards) { - return persistence.find(type, names, allowWildCards) + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return persistence.find(type, names, allowWildCards, isCaseSensitive) .stream() .filter(this::canRead) .collect(Collectors.toList()); diff --git a/stroom-documentation/stroom-documentation-impl/src/main/java/stroom/documentation/impl/DocumentationStoreImpl.java b/stroom-documentation/stroom-documentation-impl/src/main/java/stroom/documentation/impl/DocumentationStoreImpl.java index 4df6a2840bc..f0a64ec33c7 100644 --- a/stroom-documentation/stroom-documentation-impl/src/main/java/stroom/documentation/impl/DocumentationStoreImpl.java +++ b/stroom-documentation/stroom-documentation-impl/src/main/java/stroom/documentation/impl/DocumentationStoreImpl.java @@ -173,8 +173,10 @@ public Set findAssociatedNonExplorerDocRefs(DocRef docRef) { //////////////////////////////////////////////////////////////////////// @Override - public List findByNames(final List names, final boolean allowWildCards) { - return store.findByNames(names, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-event-logging-rs/stroom-event-logging-rs-impl/src/main/java/stroom/event/logging/rs/impl/RequestInfo.java b/stroom-event-logging-rs/stroom-event-logging-rs-impl/src/main/java/stroom/event/logging/rs/impl/RequestInfo.java index a35e31b8b0e..abfd5c6484e 100644 --- a/stroom-event-logging-rs/stroom-event-logging-rs-impl/src/main/java/stroom/event/logging/rs/impl/RequestInfo.java +++ b/stroom-event-logging-rs/stroom-event-logging-rs-impl/src/main/java/stroom/event/logging/rs/impl/RequestInfo.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 Crown Copyright + * Copyright 2020-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -61,7 +61,7 @@ public RequestInfo(final SecurityContext securityContext, final ContainerResourc } public static boolean objectIsLoggable(Object obj) { - return obj != null && !obj.getClass().getName().startsWith("java.") && !(obj instanceof Collection); + return obj != null && !obj.getClass().getName().startsWith("java.") && !(obj instanceof Collection); } public RequestInfo(final SecurityContext securityContext, @@ -160,10 +160,10 @@ private Object findBeforeOrAfterCallObj(Object resource, Object template) { Optional fetchMethodOptional = Arrays.stream(templateReadSupportingResource.getClass().getMethods()) - .filter(m -> m.getName().equals("fetch") - && m.getParameterCount() == 1 - && m.getParameters()[0].getType().isAssignableFrom(template.getClass())) - .findFirst(); + .filter(m -> m.getName().equals("fetch") + && m.getParameterCount() == 1 + && m.getParameters()[0].getType().isAssignableFrom(template.getClass())) + .findFirst(); if (fetchMethodOptional.isPresent()) { result = templateReadSupportingResource.fetch(template); @@ -172,7 +172,7 @@ private Object findBeforeOrAfterCallObj(Object resource, Object template) { "Unable to find appropriate fetch method for type " + template.getClass().getSimpleName()); } - } else if (resource instanceof FindWithCriteria) { + } else if (resource instanceof FindWithCriteria) { FindWithCriteria findWithCriteriaSupportingResource = (FindWithCriteria) resource; @@ -240,7 +240,7 @@ private Object findRequestObj() { paramMap = containerResourceInfo.getRequestContext().getUriInfo().getQueryParameters(false); } String paramName = paramMap.keySet().stream().findFirst().get(); - String paramValue = paramMap.get(paramName).stream().collect(Collectors.joining(", ")); + String paramValue = String.join(", ", paramMap.get(paramName)); if ("id".equals(paramName)) { return new ObjectId(paramValue); } else if ("uuid".equals(paramName)) { diff --git a/stroom-explorer/stroom-docrefinfo-api/src/main/java/stroom/docrefinfo/api/DocRefInfoService.java b/stroom-explorer/stroom-docrefinfo-api/src/main/java/stroom/docrefinfo/api/DocRefInfoService.java index e1883ed3202..32db77455ea 100644 --- a/stroom-explorer/stroom-docrefinfo-api/src/main/java/stroom/docrefinfo/api/DocRefInfoService.java +++ b/stroom-explorer/stroom-docrefinfo-api/src/main/java/stroom/docrefinfo/api/DocRefInfoService.java @@ -20,7 +20,7 @@ public interface DocRefInfoService extends DocRefDecorator { Optional name(DocRef docRef); /** - * Find by case-sensitive match on the name. + * Find by match on the name. * If allowWildCards is true '*' can be used to denote a 0-many char wild card. * Names may not be unique for a given type, so a non-wild carded nameFilter may return * more than one {@link DocRef}. @@ -31,10 +31,11 @@ public interface DocRefInfoService extends DocRefDecorator { */ List findByName(String type, String nameFilter, - boolean allowWildCards); + boolean allowWildCards, + boolean isCaseSensitive); /** - * Find by case-sensitive match on the name. + * Find by match on the name. * If allowWildCards is true '*' can be used to denote a 0-many char wild card. * Names may not be unique for a given type, so a non-wild carded nameFilter may return * more than one {@link DocRef}. Applies all nameFilters using an OR, i.e. returns all docRefs @@ -46,5 +47,6 @@ List findByName(String type, */ List findByNames(String type, List nameFilters, - boolean allowWildCards); + boolean allowWildCards, + boolean isCaseSensitive); } diff --git a/stroom-explorer/stroom-docrefinfo-mock/src/main/java/stroom/docrefinfo/mock/MockDocRefInfoService.java b/stroom-explorer/stroom-docrefinfo-mock/src/main/java/stroom/docrefinfo/mock/MockDocRefInfoService.java index 28a68a8635b..e1b71f02b45 100644 --- a/stroom-explorer/stroom-docrefinfo-mock/src/main/java/stroom/docrefinfo/mock/MockDocRefInfoService.java +++ b/stroom-explorer/stroom-docrefinfo-mock/src/main/java/stroom/docrefinfo/mock/MockDocRefInfoService.java @@ -48,14 +48,18 @@ public Optional name(final DocRef docRef) { } @Override - public List findByName(final String type, final String nameFilter, final boolean allowWildCards) { + public List findByName(final String type, + final String nameFilter, + final boolean allowWildCards, + final boolean isCaseSensitive) { return Collections.emptyList(); } @Override public List findByNames(final String type, final List nameFilters, - final boolean allowWildCards) { + final boolean allowWildCards, + final boolean isCaseSensitive) { return Collections.emptyList(); } diff --git a/stroom-explorer/stroom-explorer-impl-db/src/main/java/stroom/explorer/impl/db/ExplorerTreeDaoImpl.java b/stroom-explorer/stroom-explorer-impl-db/src/main/java/stroom/explorer/impl/db/ExplorerTreeDaoImpl.java index b432d534985..cd1547176e6 100644 --- a/stroom-explorer/stroom-explorer-impl-db/src/main/java/stroom/explorer/impl/db/ExplorerTreeDaoImpl.java +++ b/stroom-explorer/stroom-explorer-impl-db/src/main/java/stroom/explorer/impl/db/ExplorerTreeDaoImpl.java @@ -655,9 +655,10 @@ public ExplorerTreeNode findByUUID(final String uuid) { @Override public List findByNames(final List names, - final boolean allowWildCards) { + final boolean allowWildCards, + final boolean isCaseSensitive) { final Condition nameConditions = JooqUtil.createWildCardedStringsCondition( - n.UUID, names, allowWildCards, BooleanOperator.OR); + n.NAME, names, allowWildCards, BooleanOperator.OR); return JooqUtil.contextResult(explorerDbConnProvider, context -> context diff --git a/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/DocRefInfoServiceImpl.java b/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/DocRefInfoServiceImpl.java index ba5947c845b..3547c1384a3 100644 --- a/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/DocRefInfoServiceImpl.java +++ b/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/DocRefInfoServiceImpl.java @@ -84,7 +84,8 @@ public Optional name(final DocRef docRef) { @Override public List findByName(final String type, final String nameFilter, - final boolean allowWildCards) { + final boolean allowWildCards, + final boolean isCaseSensitive) { if (NullSafe.isEmptyString(nameFilter)) { return Collections.emptyList(); } else { @@ -92,13 +93,14 @@ public List findByName(final String type, if (type == null) { // No type so have to search all handlers final List result = new ArrayList<>(); - explorerActionHandlers.forEach((handlerType, handler) -> - result.addAll(handler.findByName(nameFilter, allowWildCards))); + explorerActionHandlers.forEach((handlerType, handler) -> { + result.addAll(handler.findByName(nameFilter, allowWildCards, isCaseSensitive)); + }); return result; } else { final ExplorerActionHandler handler = explorerActionHandlers.getHandler(type); Objects.requireNonNull(handler, () -> "No handler for type " + type); - return handler.findByName(nameFilter, allowWildCards); + return handler.findByName(nameFilter, allowWildCards, isCaseSensitive); } }); } @@ -107,7 +109,8 @@ public List findByName(final String type, @Override public List findByNames(final String type, final List nameFilters, - final boolean allowWildCards) { + final boolean allowWildCards, + final boolean isCaseSensitive) { Objects.requireNonNull(type); if (NullSafe.isEmptyCollection(nameFilters)) { return Collections.emptyList(); @@ -115,7 +118,7 @@ public List findByNames(final String type, return securityContextProvider.get().asProcessingUserResult(() -> { final ExplorerActionHandler handler = explorerActionHandlers.getHandler(type); Objects.requireNonNull(handler, () -> "No handler for type " + type); - return handler.findByNames(nameFilters, allowWildCards); + return handler.findByNames(nameFilters, allowWildCards, isCaseSensitive); }); } } @@ -163,7 +166,11 @@ public DocRef decorate(final DocRef docRef, // Allow decoration by name alone if feed (special case). if (FeedDoc.DOCUMENT_TYPE.equals(docRef.getType()) && docRef.getUuid() == null) { - final List list = findByName(docRef.getType(), docRef.getName(), false); + final List list = findByName( + docRef.getType(), + docRef.getName(), + false, + true); if (!NullSafe.isEmptyCollection(list)) { return list.getFirst(); } else { diff --git a/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/ExplorerServiceImpl.java b/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/ExplorerServiceImpl.java index f50950e3eb4..be6cfba84f8 100644 --- a/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/ExplorerServiceImpl.java +++ b/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/ExplorerServiceImpl.java @@ -78,6 +78,7 @@ import stroom.util.shared.PermissionException; import stroom.util.shared.ResultPage; import stroom.util.shared.UserRef; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -1781,9 +1782,9 @@ private boolean matchExpression(final List path, final ExpressionMatcher expressionMatcher, final ExpressionOperator expression) { if (node.hasNodeFlag(NodeFlag.FILTER_MATCH) && node.getDocRef() != null) { - final Map attributes = new HashMap<>(); - attributes.put(DocumentPermissionFields.DOCUMENT.getFldName(), node.getDocRef()); - attributes.put(DocumentPermissionFields.CHILDREN.getFldName(), new TermMatcher() { + final Map attributes = new HashMap<>(); + attributes.put(DocumentPermissionFields.DOCUMENT.getFldNameAsCIKey(), node.getDocRef()); + attributes.put(DocumentPermissionFields.CHILDREN.getFldNameAsCIKey(), new TermMatcher() { @Override public boolean match(final QueryField queryField, final Condition condition, @@ -1801,7 +1802,7 @@ public boolean match(final QueryField queryField, return false; } }); - attributes.put(DocumentPermissionFields.DESCENDANTS.getFldName(), new TermMatcher() { + attributes.put(DocumentPermissionFields.DESCENDANTS.getFldNameAsCIKey(), new TermMatcher() { @Override public boolean match(final QueryField queryField, final Condition condition, @@ -1818,7 +1819,7 @@ public boolean match(final QueryField queryField, return false; } }); - attributes.put(DocumentPermissionFields.USER.getFldName(), new TermMatcher() { + attributes.put(DocumentPermissionFields.USER.getFldNameAsCIKey(), new TermMatcher() { @Override public boolean match(final QueryField queryField, final Condition condition, @@ -1844,10 +1845,10 @@ public boolean match(final QueryField queryField, }; } }); - attributes.put(DocumentPermissionFields.DOCUMENT_TYPE.getFldName(), node.getDocRef().getType()); - attributes.put(DocumentPermissionFields.DOCUMENT_UUID.getFldName(), node.getDocRef().getUuid()); - attributes.put(DocumentPermissionFields.DOCUMENT_NAME.getFldName(), node.getDocRef().getName()); - attributes.put(DocumentPermissionFields.DOCUMENT_TAG.getFldName(), node.getTags()); + attributes.put(DocumentPermissionFields.DOCUMENT_TYPE.getFldNameAsCIKey(), node.getDocRef().getType()); + attributes.put(DocumentPermissionFields.DOCUMENT_UUID.getFldNameAsCIKey(), node.getDocRef().getUuid()); + attributes.put(DocumentPermissionFields.DOCUMENT_NAME.getFldNameAsCIKey(), node.getDocRef().getName()); + attributes.put(DocumentPermissionFields.DOCUMENT_TAG.getFldNameAsCIKey(), node.getTags()); return expressionMatcher.match(attributes, expression); } diff --git a/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/ExplorerTreeDao.java b/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/ExplorerTreeDao.java index 239385716b5..706af39b53d 100644 --- a/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/ExplorerTreeDao.java +++ b/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/ExplorerTreeDao.java @@ -55,7 +55,9 @@ TreeModel createModel(Function iconUrlProvider, ExplorerTreeNode findByUUID(final String uuid); - List findByNames(final List names, final boolean allowWildCards); + List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive); List findByType(final String type); } diff --git a/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/FolderExplorerActionHandler.java b/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/FolderExplorerActionHandler.java index d0ace1b1624..789eab9c49a 100644 --- a/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/FolderExplorerActionHandler.java +++ b/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/FolderExplorerActionHandler.java @@ -166,8 +166,10 @@ public void remapDependencies(final DocRef docRef, final Map rem @Override - public List findByNames(final List names, final boolean allowWildCards) { - return explorerTreeDao.findByNames(names, allowWildCards) + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return explorerTreeDao.findByNames(names, allowWildCards, isCaseSensitive) .stream() .map(ExplorerTreeNode::getDocRef) .collect(Collectors.toList()); diff --git a/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/SystemExplorerActionHandler.java b/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/SystemExplorerActionHandler.java index 3b6ed4b795c..e1119e65bd1 100644 --- a/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/SystemExplorerActionHandler.java +++ b/stroom-explorer/stroom-explorer-impl/src/main/java/stroom/explorer/impl/SystemExplorerActionHandler.java @@ -120,7 +120,9 @@ public void remapDependencies(final DocRef docRef, final Map rem @Override - public List findByNames(final List name, final boolean allowWildCards) { + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { throw new PermissionException(securityContext.getUserRef(), "You cannot perform findByNames on the System node handler"); } diff --git a/stroom-explorer/stroom-explorer-impl/src/test/java/stroom/explorer/impl/TestDocRefInfoCache.java b/stroom-explorer/stroom-explorer-impl/src/test/java/stroom/explorer/impl/TestDocRefInfoCache.java index 033c69d2560..d5cab9c3f3e 100644 --- a/stroom-explorer/stroom-explorer-impl/src/test/java/stroom/explorer/impl/TestDocRefInfoCache.java +++ b/stroom-explorer/stroom-explorer-impl/src/test/java/stroom/explorer/impl/TestDocRefInfoCache.java @@ -219,7 +219,9 @@ public Set listDocuments() { } @Override - public List findByNames(final List names, final boolean allowWildCards) { + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { return null; } diff --git a/stroom-feed/stroom-feed-impl/src/main/java/stroom/feed/impl/FeedStoreImpl.java b/stroom-feed/stroom-feed-impl/src/main/java/stroom/feed/impl/FeedStoreImpl.java index 650b50c93b4..e00a5732631 100644 --- a/stroom-feed/stroom-feed-impl/src/main/java/stroom/feed/impl/FeedStoreImpl.java +++ b/stroom-feed/stroom-feed-impl/src/main/java/stroom/feed/impl/FeedStoreImpl.java @@ -270,8 +270,10 @@ public Set findAssociatedNonExplorerDocRefs(DocRef docRef) { //////////////////////////////////////////////////////////////////////// @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-headless/src/main/java/stroom/headless/CliTranslationTaskHandler.java b/stroom-headless/src/main/java/stroom/headless/CliTranslationTaskHandler.java index 72456d7b9e0..a913e28998d 100644 --- a/stroom-headless/src/main/java/stroom/headless/CliTranslationTaskHandler.java +++ b/stroom-headless/src/main/java/stroom/headless/CliTranslationTaskHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.headless; @@ -132,7 +131,7 @@ public void exec(final InputStream dataStream, // Set the pipeline so it can be used by a filter if needed. final List pipelines = pipelineStore.findByName(feedName); - if (pipelines == null || pipelines.size() == 0) { + if (pipelines == null || pipelines.isEmpty()) { throw ProcessException.create("No pipeline found for feed name '" + feedName + "'"); } if (pipelines.size() > 1) { diff --git a/stroom-headless/src/main/java/stroom/headless/HeadlessFilter.java b/stroom-headless/src/main/java/stroom/headless/HeadlessFilter.java index 8cdba7dd651..7c9782bcdcc 100644 --- a/stroom-headless/src/main/java/stroom/headless/HeadlessFilter.java +++ b/stroom-headless/src/main/java/stroom/headless/HeadlessFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -117,7 +117,7 @@ private void writeMetaData() { try { super.startElement(URI, META_DATA, META_DATA, new AttributesImpl()); - final List keys = new ArrayList<>(metaData.keySet()); + final List keys = new ArrayList<>(metaData.stringKeySet()); // Make sure the metadata keys are in a consistent order Collections.sort(keys); diff --git a/stroom-index/stroom-index-impl-db/src/main/java/stroom/index/impl/db/IndexFieldDaoImpl.java b/stroom-index/stroom-index-impl-db/src/main/java/stroom/index/impl/db/IndexFieldDaoImpl.java index 1dac987c32b..64c751a55bd 100644 --- a/stroom-index/stroom-index-impl-db/src/main/java/stroom/index/impl/db/IndexFieldDaoImpl.java +++ b/stroom-index/stroom-index-impl-db/src/main/java/stroom/index/impl/db/IndexFieldDaoImpl.java @@ -72,11 +72,15 @@ private void ensureFieldSource(final DocRef docRef) { }); } - private Optional getFieldSource(final DocRef docRef, - final boolean lockFieldSource) { + private Optional getFieldSource(final DocRef docRef) { return JooqUtil.contextResult(queryDatasourceDbConnProvider, context -> - getFieldSource(context, docRef, lockFieldSource)); + getFieldSource(context, docRef, false)); + } + + private Optional getFieldSourceWithLock(final DSLContext context, + final DocRef docRef) { + return getFieldSource(context, docRef, true); } private Optional getFieldSource(final DSLContext context, @@ -126,7 +130,7 @@ public void addFields(final DocRef docRef, final Collection fields) JooqUtil.transaction(queryDatasourceDbConnProvider, txnContext -> { // Get a record lock on the field source, so we are the only thread // that can mutate the index fields for that source, else we can get a deadlock. - final int fieldSourceId = getFieldSource(txnContext, docRef, true) + final int fieldSourceId = getFieldSourceWithLock(txnContext, docRef) .orElseThrow(() -> new RuntimeException("No field source found for " + docRef)); // Establish which fields are already there, so we don't need to touch them. @@ -165,7 +169,8 @@ public void addFields(final DocRef docRef, final Collection fields) } LOGGER.debug("{} fields to upsert on {}", fieldCount, docRef); if (fieldCount > 0) { - // The update part doesn't update anything, intentiaonally + // The update part doesn't update anything, intentionally, as we are not changing + // records, just silently ignoring ones that are already there c.onDuplicateKeyUpdate() .set(INDEX_FIELD.FK_INDEX_FIELD_SOURCE_ID, fieldSourceId) .execute(); @@ -196,7 +201,7 @@ public void addFields(final DocRef docRef, final Collection fields) @Override public ResultPage findFields(final FindFieldCriteria criteria) { - final Optional optional = getFieldSource(criteria.getDataSourceRef(), false); + final Optional optional = getFieldSource(criteria.getDataSourceRef()); if (optional.isEmpty()) { return ResultPage.createCriterialBasedList(Collections.emptyList(), criteria); @@ -253,7 +258,7 @@ public ResultPage findFields(final FindFieldCriteria criteria) { @Override public int getFieldCount(final DocRef docRef) { - final Optional optFieldSource = getFieldSource(docRef, false); + final Optional optFieldSource = getFieldSource(docRef); if (optFieldSource.isPresent()) { return JooqUtil.contextResult(queryDatasourceDbConnProvider, context -> context .selectCount() diff --git a/stroom-index/stroom-index-impl-db/src/main/java/stroom/index/impl/db/IndexShardDaoImpl.java b/stroom-index/stroom-index-impl-db/src/main/java/stroom/index/impl/db/IndexShardDaoImpl.java index 9737c4974f3..d1d4e9c8e87 100644 --- a/stroom-index/stroom-index-impl-db/src/main/java/stroom/index/impl/db/IndexShardDaoImpl.java +++ b/stroom-index/stroom-index-impl-db/src/main/java/stroom/index/impl/db/IndexShardDaoImpl.java @@ -47,12 +47,14 @@ import stroom.query.language.functions.ValNull; import stroom.query.language.functions.ValString; import stroom.query.language.functions.ValuesConsumer; +import stroom.util.NullSafe; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.shared.PageRequest; import stroom.util.shared.Range; import stroom.util.shared.ResultPage; import stroom.util.shared.Selection; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -64,7 +66,6 @@ import org.jooq.Result; import org.jooq.impl.DSL; -import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; @@ -123,7 +124,7 @@ class IndexShardDaoImpl implements IndexShardDao { return record; }; - private static final Map> FIELD_MAP = Map.of( + private static final Map> FIELD_MAP = CIKey.mapOf( FindIndexShardCriteria.FIELD_ID, INDEX_SHARD.ID, FindIndexShardCriteria.FIELD_NODE, INDEX_SHARD.NODE_NAME, FindIndexShardCriteria.FIELD_PARTITION, INDEX_SHARD.PARTITION_NAME, @@ -215,7 +216,7 @@ public ResultPage find(final FindIndexShardCriteria criteria) { public void search(final ExpressionCriteria criteria, final FieldIndex fieldIndex, final ValuesConsumer consumer) { - final String[] fieldNames = fieldIndex.getFields(); + final List fieldNames = fieldIndex.getFieldsAsCIKeys(); final Collection> orderFields = JooqUtil.getOrderFields(FIELD_MAP, criteria); final List> dbFields = indexShardValueMapper.getDbFieldsByName(fieldNames); final Mapper[] mappers = indexShardValueMapper.getMappersForFieldNames(fieldNames); @@ -223,12 +224,12 @@ public void search(final ExpressionCriteria criteria, final Condition condition = indexShardExpressionMapper.apply(criteria.getExpression()); final boolean volumeUsed = isUsed( - Set.of(IndexShardFields.FIELD_VOLUME_PATH.getFldName(), - IndexShardFields.FIELD_VOLUME_GROUP.getFldName()), + Set.of(IndexShardFields.FIELD_VOLUME_PATH.getFldNameAsCIKey(), + IndexShardFields.FIELD_VOLUME_GROUP.getFldNameAsCIKey()), fieldNames, criteria); final boolean volumeGroupUsed = isUsed( - Set.of(IndexShardFields.FIELD_VOLUME_GROUP.getFldName()), + Set.of(IndexShardFields.FIELD_VOLUME_GROUP.getFldNameAsCIKey()), fieldNames, criteria); @@ -255,6 +256,7 @@ public void search(final ExpressionCriteria criteria, .on(INDEX_VOLUME_GROUP.ID.eq(INDEX_VOLUME.FK_INDEX_VOLUME_GROUP_ID)); } + final int fieldCount = fieldNames.size(); try (final Cursor cursor = select .where(condition) .orderBy(orderFields) @@ -264,13 +266,13 @@ public void search(final ExpressionCriteria criteria, while (cursor.hasNext()) { final Result result = cursor.fetchNext(1000); - result.forEach(r -> { - final Val[] arr = new Val[fieldNames.length]; - for (int i = 0; i < fieldNames.length; i++) { + result.forEach(rec -> { + final Val[] arr = new Val[fieldCount]; + for (int i = 0; i < fieldCount; i++) { Val val = ValNull.INSTANCE; final Mapper mapper = mappers[i]; if (mapper != null) { - val = mapper.map(r); + val = mapper.map(rec); } arr[i] = val; } @@ -388,11 +390,28 @@ public void update(final Long id, .execute()); } - private boolean isUsed(final Set fieldSet, - final String[] fields, + private boolean isUsed(final CIKey field, + final List resultFields, final ExpressionCriteria criteria) { - return Arrays.stream(fields).filter(Objects::nonNull).anyMatch(fieldSet::contains) || - ExpressionUtil.termCount(criteria.getExpression(), fieldSet) > 0; + final boolean isInResultFields = NullSafe.stream(resultFields) + .filter(Objects::nonNull) + .anyMatch(resultField -> Objects.equals(resultField, field)); + + return isInResultFields + || ExpressionUtil.termCount(criteria.getExpression(), field.get()) > 0; + } + + private boolean isUsed(final Set fieldSet, + final List resultFields, + final ExpressionCriteria criteria) { + final boolean isInResultFields = NullSafe.stream(resultFields) + .filter(Objects::nonNull) + .anyMatch(fieldSet::contains); + + return isInResultFields || ExpressionUtil.termCount( + criteria.getExpression(), + fieldSet.stream() + .map(CIKey::get).collect(Collectors.toSet())) > 0; } @@ -438,11 +457,11 @@ private Val getStatus(final byte statusPrimitive) { return ValString.create(indexShardStatus.getDisplayValue()); } - public List> getDbFieldsByName(final String[] fieldNames) { + public List> getDbFieldsByName(final List fieldNames) { return valueMapper.getDbFieldsByName(fieldNames); } - public Mapper[] getMappersForFieldNames(final String[] fieldNames) { + public Mapper[] getMappersForFieldNames(final List fieldNames) { return valueMapper.getMappersForFieldNames(fieldNames); } } @@ -480,7 +499,7 @@ private IndexShardExpressionMapper(final ExpressionMapperFactory expressionMappe } private List getIndexUuids(final List indexNames) { - return indexStore.findByNames(indexNames, true) + return indexStore.findByNames(indexNames, true, false) .stream() .map(DocRef::getUuid) .collect(Collectors.toList()); diff --git a/stroom-index/stroom-index-impl-db/src/test/java/stroom/index/impl/db/TestIndexFieldDaoImpl.java b/stroom-index/stroom-index-impl-db/src/test/java/stroom/index/impl/db/TestIndexFieldDaoImpl.java index fb4396cfbdc..15820c63f16 100644 --- a/stroom-index/stroom-index-impl-db/src/test/java/stroom/index/impl/db/TestIndexFieldDaoImpl.java +++ b/stroom-index/stroom-index-impl-db/src/test/java/stroom/index/impl/db/TestIndexFieldDaoImpl.java @@ -6,9 +6,12 @@ import stroom.datasource.api.v2.IndexField; import stroom.db.util.JooqUtil; import stroom.docref.DocRef; +import stroom.docref.StringMatch; +import stroom.docref.StringMatch.MatchType; import stroom.index.impl.IndexFieldDao; import stroom.index.shared.IndexFieldImpl; import stroom.index.shared.LuceneIndexDoc; +import stroom.test.common.TestUtil; import stroom.util.concurrent.ThreadUtil; import stroom.util.exception.ThrowingRunnable; import stroom.util.logging.LambdaLogger; @@ -18,21 +21,30 @@ import com.google.inject.Guice; import com.google.inject.Injector; +import com.google.inject.TypeLiteral; import jakarta.inject.Inject; import org.jooq.Record1; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static stroom.db.util.JooqUtil.count; @@ -225,6 +237,79 @@ void findFields() { .isEqualTo(1); } + @TestFactory + @Execution(ExecutionMode.SAME_THREAD) + Stream testCaseSensitivity() { + final AtomicBoolean hasSetupRun = new AtomicBoolean(false); + + return TestUtil.buildDynamicTestStream() + .withInputType(StringMatch.class) + .withWrappedOutputType(new TypeLiteral>() { + }) + .withTestFunction(testCase -> { + final StringMatch stringMatch = testCase.getInput(); + return getFields(DOC_REF_1, stringMatch) + .stream() + .map(IndexField::getFldName) + .collect(Collectors.toSet()); + }) + .withSimpleEqualityAssertion() + .withBeforeTestCaseAction(() -> { + if (!hasSetupRun.get()) { + + assertThat(getFields(DOC_REF_1)) + .isEmpty(); + final IndexField foo = idField("foo"); + final IndexField bar = idField("bar"); + + // Fields are not changed between tests + indexFieldDao.addFields(DOC_REF_1, List.of( + foo, + bar)); + + assertThat(getFields(DOC_REF_1)) + .hasSize(2); + + hasSetupRun.set(true); + } + }) + .addCase(StringMatch.any(), Set.of("foo", "bar")) + .addCase(StringMatch.nonNull(), Set.of("foo", "bar")) + + .addCase(StringMatch.equals("foo"), Set.of("foo")) + .addCase(StringMatch.equals("FOO"), Collections.emptySet()) + .addCase(StringMatch.equalsIgnoreCase("foo"), Set.of("foo")) + .addCase(StringMatch.equalsIgnoreCase("FOO"), Set.of("foo")) + + .addCase(StringMatch.notEquals("foo"), Set.of("bar")) + .addCase(StringMatch.notEquals("FOO"), Set.of("foo", "bar")) + .addCase(StringMatch.notEqualsIgnoreCase("foo"), Set.of("bar")) + .addCase(StringMatch.notEqualsIgnoreCase("FOO"), Set.of("bar")) + + .addCase(StringMatch.contains("oo"), Set.of("foo")) + .addCase(StringMatch.contains("OO"), Collections.emptySet()) + .addCase(StringMatch.containsIgnoreCase("oo"), Set.of("foo")) + .addCase(StringMatch.containsIgnoreCase("OO"), Set.of("foo")) + + .addCase(StringMatch.regex("^fo"), Set.of("foo")) + .addCase(StringMatch.regex("^FO"), Collections.emptySet()) + .addCase(StringMatch.regexIgnoreCase("^fo"), Set.of("foo")) + .addCase(StringMatch.regexIgnoreCase("^FO"), Set.of("foo")) + + .addCase(new StringMatch(MatchType.STARTS_WITH, true, "fo"), Set.of("foo")) + .addCase(new StringMatch(MatchType.STARTS_WITH, true, "FO"), Collections.emptySet()) + .addCase(new StringMatch(MatchType.STARTS_WITH, false, "fo"), Set.of("foo")) + .addCase(new StringMatch(MatchType.STARTS_WITH, false, "FO"), Set.of("foo")) + + .addCase(new StringMatch(MatchType.ENDS_WITH, true, "oo"), Set.of("foo")) + .addCase(new StringMatch(MatchType.ENDS_WITH, true, "OO"), Collections.emptySet()) + .addCase(new StringMatch(MatchType.ENDS_WITH, false, "oo"), Set.of("foo")) + .addCase(new StringMatch(MatchType.ENDS_WITH, false, "OO"), Set.of("foo")) + + .build(); + } + + private List getFields(final DocRef docRef) { final ResultPage resultPage = indexFieldDao.findFields( new FindFieldCriteria( @@ -233,4 +318,23 @@ private List getFields(final DocRef docRef) { docRef)); return resultPage.getValues(); } + + private List getFields(final DocRef docRef, final StringMatch stringMatch) { + final ResultPage resultPage = indexFieldDao.findFields( + new FindFieldCriteria( + new PageRequest(), + Collections.emptyList(), + docRef, + stringMatch, + null)); + return resultPage.getValues(); + } + + private static IndexField idField(final String fieldName) { + return IndexFieldImpl.builder() + .fldName(fieldName) + .fldType(FieldType.ID) + .analyzerType(AnalyzerType.NUMERIC) + .build(); + } } diff --git a/stroom-index/stroom-index-impl-db/src/test/java/stroom/index/impl/db/TestModule.java b/stroom-index/stroom-index-impl-db/src/test/java/stroom/index/impl/db/TestModule.java index 6a6e864053d..d79d5bb28b0 100644 --- a/stroom-index/stroom-index-impl-db/src/test/java/stroom/index/impl/db/TestModule.java +++ b/stroom-index/stroom-index-impl-db/src/test/java/stroom/index/impl/db/TestModule.java @@ -79,8 +79,15 @@ WordListProvider wordListProvider() { return new WordListProvider() { @Override - public List findByName(final String name) { - return List.of(); + public Set listDocuments() { + return null; + } + + @Override + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return null; } @Override diff --git a/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldCacheImpl.java b/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldCacheImpl.java index c0eeb8b4fec..e032236002a 100644 --- a/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldCacheImpl.java +++ b/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldCacheImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.index.impl; @@ -22,12 +21,14 @@ import stroom.datasource.api.v2.IndexField; import stroom.docref.DocRef; import stroom.query.common.v2.IndexFieldCache; +import stroom.query.common.v2.IndexFieldMap; import stroom.query.common.v2.IndexFieldProviders; import stroom.security.api.SecurityContext; import stroom.security.shared.DocumentPermission; import stroom.util.logging.LogUtil; import stroom.util.shared.Clearable; import stroom.util.shared.PermissionException; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -41,7 +42,7 @@ public class IndexFieldCacheImpl implements IndexFieldCache, Clearable { private static final String CACHE_NAME = "Index Field Cache"; private final IndexFieldProviders indexFieldProviders; - private final LoadingStroomCache cache; + private final LoadingStroomCache cache; private final SecurityContext securityContext; @Inject @@ -57,9 +58,9 @@ public class IndexFieldCacheImpl implements IndexFieldCache, Clearable { this::create); } - private IndexField create(final Key key) { + private IndexFieldMap create(final Key key) { return securityContext.asProcessingUserResult(() -> - indexFieldProviders.getIndexField(key.docRef, key.fieldName)); + indexFieldProviders.getIndexFields(key.docRef, key.fieldName)); } @Override @@ -74,7 +75,15 @@ public IndexField get(final DocRef docRef, final String fieldName) { LogUtil.message("You are not authorised to read {}", docRef)); } final Key key = new Key(docRef, fieldName); - return cache.get(key); + final IndexFieldMap indexFieldMap = cache.get(key); + + // Attempt to get an IndexField matching the user's input. This may throw if there are multiple + // fields with the same name (ignoring case) and none match exactly with fieldName. + if (indexFieldMap != null) { + return indexFieldMap.getClosestMatchingField(fieldName); + } else { + return null; + } } @Override @@ -82,31 +91,69 @@ public void clear() { cache.clear(); } - private static class Key { + + // -------------------------------------------------------------------------------- + + + // Pkg private for testing + static class Key { private final DocRef docRef; - private final String fieldName; + // field names are case-insensitive in stroom but may not be in the index provider + private final CIKey fieldName; + + private int hash = 0; + /** + * Set to true if the hash has been calculated and found to be zero, + * to distinguish from the default value for hash. + */ + private boolean hashIsZero = false; public Key(final DocRef docRef, final String fieldName) { this.docRef = docRef; - this.fieldName = fieldName; + this.fieldName = CIKey.of(fieldName); + } + + public DocRef getDocRef() { + return docRef; + } + + public String getFieldName() { + return fieldName.get(); } @Override - public boolean equals(final Object o) { - if (this == o) { + public boolean equals(final Object object) { + if (this == object) { return true; } - if (o == null || getClass() != o.getClass()) { + if (object == null || getClass() != object.getClass()) { return false; } - final Key key = (Key) o; - return Objects.equals(docRef, key.docRef) && Objects.equals(fieldName, key.fieldName); + final Key key = (Key) object; + return Objects.equals(docRef, key.docRef) + && Objects.equals(fieldName, key.fieldName); } @Override public int hashCode() { - return Objects.hash(docRef, fieldName); + int hash = this.hash; + // Lazily cache the hash + if (hash == 0 && !hashIsZero) { + // Case-insensitive hash + hash = Objects.hash(docRef, fieldName); + if (hash == 0) { + hashIsZero = true; + } else { + this.hash = hash; + } + } + return hash; + } + + @Override + public String toString() { + return "'" + fieldName + "' - " + docRef; } } } diff --git a/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldProvidersImpl.java b/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldProvidersImpl.java index d0d675569d3..b4357dd3a34 100644 --- a/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldProvidersImpl.java +++ b/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldProvidersImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,19 +12,19 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.index.impl; -import stroom.datasource.api.v2.IndexField; import stroom.docref.DocRef; +import stroom.query.common.v2.IndexFieldMap; import stroom.query.common.v2.IndexFieldProvider; import stroom.query.common.v2.IndexFieldProviders; import stroom.security.api.SecurityContext; import stroom.security.shared.DocumentPermission; import stroom.util.logging.LogUtil; import stroom.util.shared.PermissionException; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -37,6 +37,7 @@ @Singleton public class IndexFieldProvidersImpl implements IndexFieldProviders { + // DocRef.type => IndexFieldProvider private final Map providers = new HashMap<>(); private final SecurityContext securityContext; @@ -50,10 +51,11 @@ public class IndexFieldProvidersImpl implements IndexFieldProviders { } @Override - public IndexField getIndexField(final DocRef docRef, final String fieldName) { + public IndexFieldMap getIndexFields(final DocRef docRef, final CIKey fieldName) { Objects.requireNonNull(docRef, "Null DocRef supplied"); Objects.requireNonNull(docRef.getType(), "Null DocRef type supplied"); Objects.requireNonNull(fieldName, "Null field name supplied"); + Objects.requireNonNull(fieldName.get(), "Null field name supplied"); if (!securityContext.hasDocumentPermission(docRef, DocumentPermission.USE)) { throw new PermissionException( @@ -65,6 +67,6 @@ public IndexField getIndexField(final DocRef docRef, final String fieldName) { throw new NullPointerException("No provider can be found for: " + docRef.getType()); } - return provider.getIndexField(docRef, fieldName); + return provider.getIndexFields(docRef, fieldName); } } diff --git a/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldServiceImpl.java b/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldServiceImpl.java index 29fd89fece7..3347f7af828 100644 --- a/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldServiceImpl.java +++ b/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexFieldServiceImpl.java @@ -21,6 +21,7 @@ import stroom.docref.DocRef; import stroom.docref.StringMatch; import stroom.index.shared.LuceneIndexDoc; +import stroom.query.common.v2.IndexFieldMap; import stroom.security.api.SecurityContext; import stroom.security.shared.DocumentPermission; import stroom.util.NullSafe; @@ -28,6 +29,7 @@ import stroom.util.logging.LambdaLoggerFactory; import stroom.util.shared.PageRequest; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -83,8 +85,7 @@ public void transferFieldsToDB(final DocRef docRef) { final IndexStore indexStore = indexStoreProvider.get(); final LuceneIndexDoc index = indexStore.readDocument(docRef); if (index != null) { - final List fields = NullSafe.list(index.getFields()) - .stream() + final List fields = NullSafe.stream(index.getFields()) .map(field -> (IndexField) field) .toList(); addFields(docRef, fields); @@ -112,7 +113,7 @@ public int getFieldCount(final DocRef docRef) { } @Override - public IndexField getIndexField(final DocRef docRef, final String fieldName) { + public IndexFieldMap getIndexFields(final DocRef docRef, final CIKey fieldName) { return securityContext.useAsReadResult(() -> { // Check for read permission. @@ -125,13 +126,17 @@ public IndexField getIndexField(final DocRef docRef, final String fieldName) { PageRequest.oneRow(), null, docRef, - StringMatch.equals(fieldName, true), + StringMatch.equalsIgnoreCase(fieldName.get()), null); + + // Get all fields regardless of case final ResultPage resultPage = findFields(findIndexFieldCriteria); - if (resultPage.size() > 0) { - return resultPage.getFirst(); + + if (!resultPage.isEmpty()) { + return IndexFieldMap.fromFieldList(fieldName, resultPage.getValues()); + } else { + return null; } - return null; }); } diff --git a/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexStoreImpl.java b/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexStoreImpl.java index a4c5d62a33e..0369926a08a 100644 --- a/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexStoreImpl.java +++ b/stroom-index/stroom-index-impl/src/main/java/stroom/index/impl/IndexStoreImpl.java @@ -237,8 +237,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-index/stroom-index-impl/src/test/java/stroom/index/impl/TestIndexFieldCacheImpl.java b/stroom-index/stroom-index-impl/src/test/java/stroom/index/impl/TestIndexFieldCacheImpl.java new file mode 100644 index 00000000000..9a7abd2c0e9 --- /dev/null +++ b/stroom-index/stroom-index-impl/src/test/java/stroom/index/impl/TestIndexFieldCacheImpl.java @@ -0,0 +1,72 @@ +package stroom.index.impl; + +import stroom.docref.DocRef; +import stroom.index.impl.IndexFieldCacheImpl.Key; +import stroom.test.common.TestUtil; + +import io.vavr.Tuple; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; + +import java.util.Objects; +import java.util.stream.Stream; + +class TestIndexFieldCacheImpl { + + @TestFactory + Stream test() { + + final DocRef docRef1 = DocRef.builder() + .type("type") + .name("docref1") + .randomUuid() + .build(); + final DocRef docRef2 = DocRef.builder() + .type("type") + .name("docref2") + .randomUuid() + .build(); + + return TestUtil.buildDynamicTestStream() + .withInputTypes(Key.class, Key.class) + .withOutputType(boolean.class) + .withTestFunction(testCase -> { + + final boolean areObjectsEqual = Objects.equals( + testCase.getInput()._1(), + testCase.getInput()._2()); + final boolean areHashesEqual = Objects.equals( + testCase.getInput()._1().hashCode(), + testCase.getInput()._2().hashCode()); + + Assertions.assertThat(areHashesEqual) + .isEqualTo(areObjectsEqual); + + return areObjectsEqual; + }) + .withSimpleEqualityAssertion() + .addCase( + Tuple.of(new Key(docRef1, null), new Key(docRef1, null)), + true) + .addCase( + Tuple.of(new Key(docRef1, ""), new Key(docRef1, "")), + true) + .addCase( + Tuple.of(new Key(docRef1, "bar"), new Key(docRef1, "bar")), + true) + .addCase( + Tuple.of(new Key(docRef1, "bar"), new Key(docRef1, "BAR")), + true) + .addCase( + Tuple.of(new Key(docRef1, "foo"), new Key(docRef1, "BAR")), + false) + .addCase( + Tuple.of(new Key(docRef1, "bar"), new Key(docRef2, "bar")), + false) + .addCase( + Tuple.of(new Key(docRef1, ""), new Key(docRef2, "bar")), + false) + .build(); + } +} diff --git a/stroom-index/stroom-index-lucene553/src/main/java/stroom/index/lucene553/SearchExpressionQueryBuilder.java b/stroom-index/stroom-index-lucene553/src/main/java/stroom/index/lucene553/SearchExpressionQueryBuilder.java index a1e94b8e55a..0f15d7ca8d1 100644 --- a/stroom-index/stroom-index-lucene553/src/main/java/stroom/index/lucene553/SearchExpressionQueryBuilder.java +++ b/stroom-index/stroom-index-lucene553/src/main/java/stroom/index/lucene553/SearchExpressionQueryBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.index.lucene553; @@ -31,6 +30,7 @@ import stroom.query.common.v2.DateExpressionParser; import stroom.query.common.v2.IndexFieldCache; import stroom.search.impl.SearchException; +import stroom.util.NullSafe; import org.apache.lucene553.analysis.Analyzer; import org.apache.lucene553.index.Term; @@ -231,22 +231,14 @@ private Query getQuery(final ExpressionItem item, private Query getTermQuery(final ExpressionTerm term, final Set terms) { - String field = term.getField(); + // Clean strings to remove unwanted whitespace that the user may have added accidentally. + final String field = NullSafe.trim(term.getField()); final Condition condition = term.getCondition(); - String value = term.getValue(); + final String value = NullSafe.trim(term.getValue()); final DocRef docRef = term.getDocRef(); - // Clean strings to remove unwanted whitespace that the user may have - // added accidentally. - if (field != null) { - field = field.trim(); - } - if (value != null) { - value = value.trim(); - } - // Try and find the referenced field. - if (field == null || field.isEmpty()) { + if (field.isEmpty()) { throw new SearchException("Field not set"); } final IndexField indexField = indexFieldCache.get(indexDocRef, field); @@ -265,12 +257,11 @@ private Query getTermQuery(final ExpressionTerm term, throw new SearchException("Doc Ref not set for field: " + field); } } else { - if (value == null || value.isEmpty()) { + if (value.isEmpty()) { return null; } } - // Create a query based on the field type and condition. if (FieldType.INTEGER.equals(indexField.getFldType())) { switch (condition) { diff --git a/stroom-index/stroom-index-lucene553/src/test/java/stroom/index/lucene553/TestSearchExpressionQueryBuilder.java b/stroom-index/stroom-index-lucene553/src/test/java/stroom/index/lucene553/TestSearchExpressionQueryBuilder.java index 1aa4ed2b50c..082d9844d2f 100644 --- a/stroom-index/stroom-index-lucene553/src/test/java/stroom/index/lucene553/TestSearchExpressionQueryBuilder.java +++ b/stroom-index/stroom-index-lucene553/src/test/java/stroom/index/lucene553/TestSearchExpressionQueryBuilder.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Optional; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; @@ -57,7 +58,14 @@ private void test(AnalyzerType analyzerType) { final WordListProvider wordListProvider = new WordListProvider() { @Override - public List findByName(final String name) { + public Set listDocuments() { + return Set.of(dictionaryRef); + } + + @Override + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { return List.of(dictionaryRef); } diff --git a/stroom-index/stroom-index-lucene980/src/main/java/stroom/index/lucene980/Lucene980MemoryIndex.java b/stroom-index/stroom-index-lucene980/src/main/java/stroom/index/lucene980/Lucene980MemoryIndex.java index 5f4215deeea..2ea8e606af6 100644 --- a/stroom-index/stroom-index-lucene980/src/main/java/stroom/index/lucene980/Lucene980MemoryIndex.java +++ b/stroom-index/stroom-index-lucene980/src/main/java/stroom/index/lucene980/Lucene980MemoryIndex.java @@ -1,16 +1,35 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.index.lucene980; import stroom.datasource.api.v2.IndexField; +import stroom.docref.DocRef; import stroom.index.lucene980.SearchExpressionQueryBuilder.SearchExpressionQuery; import stroom.index.lucene980.analyser.AnalyzerFactory; import stroom.index.shared.LuceneIndexField; import stroom.query.api.v2.ExpressionUtil; import stroom.query.api.v2.SearchRequest; -import stroom.query.common.v2.IndexFieldCache; +import stroom.query.common.v2.IndexFieldMap; import stroom.search.extraction.FieldValue; import stroom.search.impl.SearchException; +import stroom.util.NullSafe; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import org.apache.lucene980.analysis.Analyzer; @@ -20,11 +39,14 @@ import org.apache.lucene980.search.TopDocs; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; class Lucene980MemoryIndex implements stroom.search.extraction.MemoryIndex { @@ -32,9 +54,13 @@ class Lucene980MemoryIndex implements stroom.search.extraction.MemoryIndex { private final SearchExpressionQueryBuilderFactory searchExpressionQueryBuilderFactory; private final Map analyzerMap = new HashMap<>(); - private Map> cachedFields; + // private Map> cachedFields; + private Map cachedFields; private SearchExpressionQuery cachedQuery; + private Set caseSenseExpressionFields; + private Set expressionFields; + @Inject public Lucene980MemoryIndex(final SearchExpressionQueryBuilderFactory searchExpressionQueryBuilderFactory) { this.searchExpressionQueryBuilderFactory = searchExpressionQueryBuilderFactory; @@ -43,33 +69,81 @@ public Lucene980MemoryIndex(final SearchExpressionQueryBuilderFactory searchExpr @Override public boolean match(final SearchRequest searchRequest, final List fieldValues) { // Instantiate the cached fields we care about. - if (cachedFields == null) { - final List fields = ExpressionUtil.fields(searchRequest.getQuery().getExpression()); - cachedFields = new HashMap<>(); - fields.forEach(field -> cachedFields.put(field, Optional.empty())); + if (expressionFields == null) { + // These may be case-sensitive if there are >1 IndexFields with the same name (ignoring case) + caseSenseExpressionFields = new HashSet<>(ExpressionUtil.fields(searchRequest.getQuery().getExpression())); + expressionFields = caseSenseExpressionFields.stream() + .map(CIKey::of) + .collect(Collectors.toSet()); +// cachedFields = new HashMap<>(); +// searchRequestFields.stream() +// .map(CIKey::of) +// .forEach(field -> +// cachedFields.put(field, IndexFieldMap.empty(field))); } // OPTIMISATION: If we have no fields in the expression then match. - if (cachedFields.isEmpty()) { + if (expressionFields.isEmpty()) { return true; } MemoryIndex memoryIndex = null; for (final FieldValue fieldValue : fieldValues) { final IndexField indexField = fieldValue.field(); - final LuceneIndexField luceneIndexField = LuceneIndexField - .fromIndexField(indexField); - - final Optional cachedField = cachedFields.get(indexField.getFldName()); - // Ignore fields we don't need. - if (cachedField != null) { - if (!Objects.equals(cachedField.orElse(null), indexField)) { - // We are adding a field. - cachedFields.put(indexField.getFldName(), Optional.of(indexField)); - // Since we changed the fields we will need to recreate the query. + final CIKey caseInsenseFieldName = CIKey.of(indexField.getFldName()); + + // Ignore fields that are not in the search expr + if (!expressionFields.contains(caseInsenseFieldName)) { + // Merge this field into the cache of IndexFieldMaps + cachedFields.compute(caseInsenseFieldName, (key, currIndexFieldMap) -> { + if (currIndexFieldMap == null) { + return IndexFieldMap.forSingleField(key, indexField); + } else { + return IndexFieldMap.merge( + currIndexFieldMap, + IndexFieldMap.forSingleField(key, indexField)); + } + }); + + // All this is to allow for the idx having fields like 'foo' and 'FOO' + final IndexFieldMap indexFieldMap = cachedFields.get(caseInsenseFieldName); + boolean hasCacheChanged = false; + if (indexFieldMap == null) { + // Nothing matching this case insense name + cachedFields.put( + caseInsenseFieldName, + IndexFieldMap.forSingleField(caseInsenseFieldName, indexField)); + hasCacheChanged = true; + } else { + final IndexField existingField = indexFieldMap.getExactMatchingField(indexField.getFldName()); + if (existingField == null) { + // This exact field is not present so merge this field in to the map + cachedFields.put( + caseInsenseFieldName, + IndexFieldMap.merge( + indexFieldMap, + IndexFieldMap.forSingleField(caseInsenseFieldName, indexField))); + hasCacheChanged = true; + } else if (!Objects.equals(existingField, indexField)) { + // Field is different, so we need to update it in the cache + final List fields = new ArrayList<>(); + indexFieldMap.getFields() + .stream() + .filter(field -> !field.getFldName().equals(indexField.getFldName())) + .forEach(fields::add); + fields.add(indexField); + cachedFields.put( + caseInsenseFieldName, + IndexFieldMap.fromFieldList(caseInsenseFieldName, fields)); + hasCacheChanged = true; + } + } + if (hasCacheChanged) { cachedQuery = null; } + final LuceneIndexField luceneIndexField = LuceneIndexField + .fromIndexField(indexField); // If the field is indexed then add it to the in memory index. if (luceneIndexField.isIndexed()) { final Analyzer fieldAnalyzer = getAnalyser(luceneIndexField); @@ -119,12 +193,12 @@ private SearchExpressionQuery getQuery(final SearchRequest searchRequest) { try { // We will cache the query until the fields change. if (cachedQuery == null) { - final IndexFieldCache indexFieldCache = (key, fieldName) -> cachedFields.get(fieldName).orElse(null); final SearchExpressionQueryBuilder searchExpressionQueryBuilder = searchExpressionQueryBuilderFactory.create( searchRequest.getQuery().getDataSource(), - indexFieldCache, + this::getIndexField, searchRequest.getDateTimeSettings()); + cachedQuery = searchExpressionQueryBuilder .buildQuery(searchRequest.getQuery().getExpression()); } @@ -135,12 +209,24 @@ private SearchExpressionQuery getQuery(final SearchRequest searchRequest) { } } + private IndexField getIndexField(final DocRef docRef, final String fieldName) { + if (cachedFields != null) { + return NullSafe.get( + cachedFields.get(CIKey.of(fieldName)), + indexFieldMap -> + indexFieldMap.getClosestMatchingField(fieldName)); + } else { + return null; + } + } + private Analyzer getAnalyser(final LuceneIndexField indexField) { try { Analyzer fieldAnalyzer = analyzerMap.get(indexField.getFldName()); if (fieldAnalyzer == null) { // Add the field analyser. - fieldAnalyzer = AnalyzerFactory.create(indexField.getAnalyzerType(), + fieldAnalyzer = AnalyzerFactory.create( + indexField.getAnalyzerType(), indexField.isCaseSensitive()); analyzerMap.put(indexField.getFldName(), fieldAnalyzer); } diff --git a/stroom-index/stroom-index-lucene980/src/main/java/stroom/index/lucene980/SearchExpressionQueryBuilder.java b/stroom-index/stroom-index-lucene980/src/main/java/stroom/index/lucene980/SearchExpressionQueryBuilder.java index bd945a4d664..9232ed811aa 100644 --- a/stroom-index/stroom-index-lucene980/src/main/java/stroom/index/lucene980/SearchExpressionQueryBuilder.java +++ b/stroom-index/stroom-index-lucene980/src/main/java/stroom/index/lucene980/SearchExpressionQueryBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.index.lucene980; @@ -31,6 +30,7 @@ import stroom.query.common.v2.DateExpressionParser; import stroom.query.common.v2.IndexFieldCache; import stroom.search.impl.SearchException; +import stroom.util.NullSafe; import org.apache.lucene980.analysis.Analyzer; import org.apache.lucene980.document.DoubleField; @@ -223,28 +223,19 @@ private Query getQuery(final ExpressionItem item, } } } - return null; } private Query getTermQuery(final ExpressionTerm term, final Set terms) { - String field = term.getField(); + // Clean strings to remove unwanted whitespace that the user may have added accidentally. + final String field = NullSafe.trim(term.getField()); final Condition condition = term.getCondition(); - String value = term.getValue(); + final String value = NullSafe.trim(term.getValue()); final DocRef docRef = term.getDocRef(); - // Clean strings to remove unwanted whitespace that the user may have - // added accidentally. - if (field != null) { - field = field.trim(); - } - if (value != null) { - value = value.trim(); - } - // Try and find the referenced field. - if (field == null || field.isEmpty()) { + if (field.isEmpty()) { throw new SearchException("Field not set"); } final IndexField indexField = indexFieldCache.get(indexDocRef, field); @@ -263,12 +254,11 @@ private Query getTermQuery(final ExpressionTerm term, throw new SearchException("Doc Ref not set for field: " + field); } } else { - if (value == null || value.isEmpty()) { + if (value.isEmpty()) { return null; } } - // Create a query based on the field type and condition. if (FieldType.INTEGER.equals(indexField.getFldType())) { switch (condition) { diff --git a/stroom-index/stroom-index-lucene980/src/test/java/stroom/index/lucene980/TestSearchExpressionQueryBuilder.java b/stroom-index/stroom-index-lucene980/src/test/java/stroom/index/lucene980/TestSearchExpressionQueryBuilder.java index 111d48c698d..7599c2f6744 100644 --- a/stroom-index/stroom-index-lucene980/src/test/java/stroom/index/lucene980/TestSearchExpressionQueryBuilder.java +++ b/stroom-index/stroom-index-lucene980/src/test/java/stroom/index/lucene980/TestSearchExpressionQueryBuilder.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Optional; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; @@ -57,8 +58,15 @@ private void test(AnalyzerType analyzerType) { final WordListProvider wordListProvider = new WordListProvider() { @Override - public List findByName(final String name) { - return List.of(dictionaryRef); + public Set listDocuments() { + return null; + } + + @Override + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return null; } @Override diff --git a/stroom-index/stroom-index-mock/src/main/java/stroom/index/mock/MockIndexFieldService.java b/stroom-index/stroom-index-mock/src/main/java/stroom/index/mock/MockIndexFieldService.java index a710f1206e9..fa9b26b9290 100644 --- a/stroom-index/stroom-index-mock/src/main/java/stroom/index/mock/MockIndexFieldService.java +++ b/stroom-index/stroom-index-mock/src/main/java/stroom/index/mock/MockIndexFieldService.java @@ -23,11 +23,13 @@ import stroom.index.impl.IndexFieldService; import stroom.index.impl.IndexStore; import stroom.index.shared.LuceneIndexDoc; +import stroom.query.common.v2.IndexFieldMap; import stroom.util.NullSafe; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.shared.PageRequest; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import stroom.util.string.StringMatcher; import jakarta.inject.Inject; @@ -115,18 +117,20 @@ public int getFieldCount(final DocRef dataSourceRef) { } @Override - public IndexField getIndexField(final DocRef docRef, final String fieldName) { + public IndexFieldMap getIndexFields(final DocRef docRef, final CIKey fieldName) { final FindFieldCriteria findIndexFieldCriteria = new FindFieldCriteria( PageRequest.oneRow(), null, docRef, - StringMatch.equals(fieldName), + StringMatch.equalsIgnoreCase(fieldName.get()), null); final ResultPage resultPage = findFields(findIndexFieldCriteria); + if (resultPage.size() > 0) { - return resultPage.getFirst(); + return IndexFieldMap.fromFieldList(fieldName, resultPage.getValues()); + } else { + return null; } - return null; } @Override diff --git a/stroom-job/stroom-job-impl-db/src/main/java/stroom/job/impl/db/JobDaoImpl.java b/stroom-job/stroom-job-impl-db/src/main/java/stroom/job/impl/db/JobDaoImpl.java index 54c391346c0..1887d93ddf6 100644 --- a/stroom-job/stroom-job-impl-db/src/main/java/stroom/job/impl/db/JobDaoImpl.java +++ b/stroom-job/stroom-job-impl-db/src/main/java/stroom/job/impl/db/JobDaoImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import stroom.job.shared.Job; import stroom.util.shared.HasIntCrud; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.validation.constraints.NotNull; @@ -55,7 +56,7 @@ public class JobDaoImpl implements JobDao, HasIntCrud { // private static final Logger LOGGER = LoggerFactory.getLogger(JobDao.class); - private static final Map> FIELD_MAP = Map.of( + private static final Map> FIELD_MAP = CIKey.mapOf( FindJobCriteria.FIELD_ID, JOB.ID, FindJobCriteria.FIELD_NAME, JOB.NAME); diff --git a/stroom-job/stroom-job-impl-db/src/main/java/stroom/job/impl/db/JobNodeDaoImpl.java b/stroom-job/stroom-job-impl-db/src/main/java/stroom/job/impl/db/JobNodeDaoImpl.java index 471a7bba03c..4ba93120392 100644 --- a/stroom-job/stroom-job-impl-db/src/main/java/stroom/job/impl/db/JobNodeDaoImpl.java +++ b/stroom-job/stroom-job-impl-db/src/main/java/stroom/job/impl/db/JobNodeDaoImpl.java @@ -31,6 +31,7 @@ import stroom.util.logging.LambdaLoggerFactory; import stroom.util.shared.HasIntCrud; import stroom.util.shared.scheduler.Schedule; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.validation.constraints.NotNull; @@ -56,7 +57,7 @@ public class JobNodeDaoImpl implements JobNodeDao, HasIntCrud { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(JobNodeDaoImpl.class); - private static final Map> FIELD_MAP = Map.of( + private static final Map> FIELD_MAP = CIKey.mapOf( FindJobNodeCriteria.FIELD_ID_ID, JOB_NODE.ID, FindJobNodeCriteria.FIELD_JOB_NAME, JOB.NAME, FindJobNodeCriteria.FIELD_ID_NODE, JOB_NODE.NODE_NAME, diff --git a/stroom-kafka/stroom-kafka-impl/src/main/java/stroom/kafka/impl/KafkaConfigStoreImpl.java b/stroom-kafka/stroom-kafka-impl/src/main/java/stroom/kafka/impl/KafkaConfigStoreImpl.java index 6c7751b8a64..7fa5158ab22 100644 --- a/stroom-kafka/stroom-kafka-impl/src/main/java/stroom/kafka/impl/KafkaConfigStoreImpl.java +++ b/stroom-kafka/stroom-kafka-impl/src/main/java/stroom/kafka/impl/KafkaConfigStoreImpl.java @@ -209,8 +209,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-meta/build.gradle b/stroom-meta/build.gradle index 315694638ad..b6da8509dff 100644 --- a/stroom-meta/build.gradle +++ b/stroom-meta/build.gradle @@ -1 +1,3 @@ ext.moduleName = 'stroom.meta' + + diff --git a/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/AttributeMap.java b/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/AttributeMap.java index 92f09750e3d..5cd9ca2218c 100644 --- a/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/AttributeMap.java +++ b/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/AttributeMap.java @@ -1,28 +1,56 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.meta.api; import stroom.util.NullSafe; import stroom.util.date.DateUtil; +import stroom.util.shared.GwtNullSafe; +import stroom.util.shared.string.CIKey; import java.time.Instant; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.BinaryOperator; +import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collector; +import java.util.stream.Collectors; /** - * Map that does not care about key case. + * Mutable map that does not care about key case. */ -public class AttributeMap extends CIStringHashMap { +public class AttributeMap implements Map { private final boolean overrideEmbeddedMeta; + private final Map map = new HashMap<>(); public AttributeMap(final boolean overrideEmbeddedMeta) { this.overrideEmbeddedMeta = overrideEmbeddedMeta; } - public AttributeMap(final boolean overrideEmbeddedMeta, final Map values) { + public AttributeMap(final boolean overrideEmbeddedMeta, + final Map values) { this.overrideEmbeddedMeta = overrideEmbeddedMeta; putAll(values); } @@ -31,7 +59,7 @@ public AttributeMap() { this.overrideEmbeddedMeta = false; } - public AttributeMap(final Map values) { + public AttributeMap(final Map values) { this.overrideEmbeddedMeta = false; putAll(values); } @@ -41,14 +69,184 @@ private AttributeMap(final Builder builder) { putAll(builder.attributes); } + /** + * @return An unmodifiable view of the underlying map. + */ + public Map asUnmodifiableStringKeyedMap() { + return entrySet().stream() + .collect(Collectors.toUnmodifiableMap(entry -> entry.getKey().get(), Entry::getValue)); + } + + /** + * @return An unmodifiable view of the underlying map. + */ + public Map asUnmodifiableMap() { + return Collections.unmodifiableMap(map); + } + + public Set> stringEntrySet() { + return entrySet().stream() + .map(entry -> Map.entry(entry.getKey().get(), entry.getValue())) + .collect(Collectors.toSet()); + } + + public Set stringKeySet() { + return keySet() + .stream() + .map(CIKey::get) + .collect(Collectors.toSet()); + } + + @Override + public int size() { + return map.size(); + } + + @Override + public boolean isEmpty() { + return map.isEmpty(); + } + @Override + public void clear() { + map.clear(); + } + public String put(final String key, final String value) { - if (key != null && StandardHeaderArguments.DATE_HEADER_KEYS.contains(key)) { - final String normalisedValue = DateUtil.normaliseDate(value, true); - return super.put(key, normalisedValue); - } else { - return super.put(key, value); + // CIString used to trim all keys + return put(CIKey.trimmed(key), value); + } + + @Override + public String put(final CIKey key, final String value) { + Objects.requireNonNull(key); + String normalisedValue = value; + if (StandardHeaderArguments.DATE_HEADER_KEYS.contains(key)) { + normalisedValue = DateUtil.normaliseDate(value, true); + } + if (normalisedValue != null) { + normalisedValue = normalisedValue.trim(); } + return map.put(key, normalisedValue); + } + + @Override + public void putAll(final Map map) { + if (NullSafe.hasEntries(map)) { + // Delegate to put so its extra logic gets called + map.forEach(this::put); + } + } + + public void putAll(final AttributeMap attributeMap) { + if (attributeMap != null) { + // Delegate to put so its extra logic gets called + attributeMap.map.forEach(this::put); + } + } + + @Deprecated // Should be using String or CIKey + @Override + public String get(final Object key) { + return map.get(key); + } + + // Make it clearer that we expect a CIKey rather than an Object + public String get(final CIKey key) { + return map.get(key); + } + + /** + * Equivalent to calling + * 
{@code get(CIKey.ofTrimmed(key))}
+ */ + // Overload all the super methods that take key as an Object as the compiler + // won't spot people using a String key. + public String get(final String key) { + // CIString used to trim all keys + return map.get(CIKey.trimmed(key)); + } + + @Deprecated // Should be using String or CIKey + @Override + public String getOrDefault(final Object key, final String defaultValue) { + return map.getOrDefault(key, defaultValue); + } + + public String getOrDefault(final String key, final String defaultValue) { + // CIString used to trim all keys + return map.getOrDefault(CIKey.trimmed(key), defaultValue); + } + + public String getOrDefault(final CIKey key, final String defaultValue) { + return map.getOrDefault(key, defaultValue); + } + + public boolean remove(final String key, final String value) { + // CIString used to trim all keys + return map.remove(CIKey.trimmed(key), value); + } + + @Deprecated // Should be using String or CIKey + @Override + public String remove(final Object key) { + return map.remove(key); + } + + public String remove(final String key) { + // CIString used to trim all keys + return map.remove(CIKey.trimmed(key)); + } + + // Make it clearer that we expect a CIKey rather than an Object + public String remove(final CIKey key) { + return map.remove(key); + } + + @Deprecated // Should be using String or CIKey + @Override + public boolean containsKey(final Object key) { + return map.containsKey(key); + } + + // Make it clearer that we expect a CIKey rather than an Object + public boolean containsKey(final CIKey key) { + return map.containsKey(key); + } + + public boolean containsKey(final String key) { + // CIString used to trim all keys + return map.containsKey(CIKey.trimmed(key)); + } + + @Deprecated // Should be using String + @Override + public boolean containsValue(final Object value) { + return map.containsValue(value); + } + + public boolean containsValue(final String value) { + return map.containsValue(value); + } + + @Override + public void forEach(final BiConsumer action) { + map.forEach(action); + } + + /** + * Equivalent to {@link Map#putAll(Map)} but for a map keyed by strings. + */ + public void putAllWithStringKeys(final Map map) { + GwtNullSafe.map(map) + .forEach(this::put); + } + + public Set keySetAsStrings() { + return map.keySet() + .stream() + .map(CIKey::get) + .collect(Collectors.toSet()); } /** @@ -57,9 +255,9 @@ public String put(final String key, final String value) { * * @return The previous value for the key. */ - public String putCurrentDateTime(final String key) { - // Already normalised, so use super.put not the local one - return super.put(key, DateUtil.createNormalDateTimeString()); + public String putCurrentDateTime(final CIKey key) { + // Already normalised, so use map.put not the local one + return map.put(key, DateUtil.createNormalDateTimeString()); } /** @@ -68,10 +266,10 @@ public String putCurrentDateTime(final String key) { * * @return The previous value for the key. */ - public String putDateTime(final String key, final Long epochMs) { + public String putDateTime(final CIKey key, final Long epochMs) { final String dateStr = DateUtil.createNormalDateTimeString(epochMs); - // Already normalised, so use super.put not the local one - return super.put(key, dateStr); + // Already normalised, so use map.put not the local one + return map.put(key, dateStr); } /** @@ -80,13 +278,13 @@ public String putDateTime(final String key, final Long epochMs) { * * @return The previous value for the key. */ - public String putDateTime(final String key, final Instant instant) { + public String putDateTime(final CIKey key, final Instant instant) { if (instant == null) { - return super.put(key, null); + return map.put(key, null); } else { final String dateStr = DateUtil.createNormalDateTimeString(instant.toEpochMilli()); - // Already normalised, so use super.put not the local one - return super.put(key, dateStr); + // Already normalised, so use map.put not the local one + return map.put(key, dateStr); } } @@ -96,9 +294,9 @@ public String putDateTime(final String key, final Instant instant) { * * @return The previous value for the key. */ - public String appendDateTime(final String key, final Instant instant) { + public String appendDateTime(final CIKey key, final Instant instant) { if (instant != null) { - String val = super.get(key); + String val = map.get(key); final String dateStr = DateUtil.createNormalDateTimeString(instant.toEpochMilli()); if (NullSafe.isEmptyString(val)) { val = dateStr; @@ -108,7 +306,7 @@ public String appendDateTime(final String key, final Instant instant) { } val += dateStr; } - return super.put(key, val); + return map.put(key, val); } else { return null; } @@ -120,13 +318,13 @@ public String appendDateTime(final String key, final Instant instant) { * * @return The previous value for the key. */ - public String appendItem(final String key, final String item) { + public String appendItem(final CIKey key, final String item) { if (item != null) { String normalisedItem = item; if (key != null && StandardHeaderArguments.DATE_HEADER_KEYS.contains(key)) { normalisedItem = DateUtil.normaliseDate(item, true); } - String val = super.get(key); + String val = map.get(key); if (NullSafe.isEmptyString(val)) { val = normalisedItem; } else { @@ -135,7 +333,7 @@ public String appendItem(final String key, final String item) { } val += normalisedItem; } - return super.put(key, val); + return map.put(key, val); } else { return null; } @@ -151,7 +349,7 @@ public String appendItem(final String key, final String item) { * null if there is no entry for the key. * @return The previous value for the key. */ - public String appendItemIf(final String key, + public String appendItemIf(final CIKey key, final String item, final Predicate currentValuePredicate) { if (item != null) { @@ -159,7 +357,7 @@ public String appendItemIf(final String key, if (key != null && StandardHeaderArguments.DATE_HEADER_KEYS.contains(key)) { normalisedItem = DateUtil.normaliseDate(item, true); } - String val = super.get(key); + String val = map.get(key); Objects.requireNonNull(currentValuePredicate); if (currentValuePredicate.test(val)) { if (NullSafe.isEmptyString(val)) { @@ -170,7 +368,7 @@ public String appendItemIf(final String key, } val += normalisedItem; } - return super.put(key, val); + return map.put(key, val); } else { return val; } @@ -182,24 +380,55 @@ public String appendItemIf(final String key, /** * Put an entry where the value is itself a collection of values, e.g. a list of files */ - public String putCollection(final String key, Collection values) { + public String putCollection(final CIKey key, Collection values) { final String value; if (values == null) { value = null; } else if (values.isEmpty()) { value = ""; } else { - value = String.join(AttributeMapUtil.VALUE_DELIMITER, values); + value = values.stream() + .filter(Objects::nonNull) + .map(String::trim) + .collect(Collectors.joining(AttributeMapUtil.VALUE_DELIMITER)); } return put(key, value); } + public Set keySet() { + return map.keySet(); + } + + public Collection values() { + return map.values(); + } + + public Set> entrySet() { + return map.entrySet(); + } + + public Set> entrySetAsString() { + return map.entrySet() + .stream() + .map(entry -> Map.entry(entry.getKey().get(), entry.getValue())) + .collect(Collectors.toSet()); + } + + public String putIfAbsent(final CIKey key, final String value) { + return map.putIfAbsent(key, value); + } + + public String computeIfAbsent(final CIKey key, + final Function mappingFunction) { + return map.computeIfAbsent(key, mappingFunction); + } + /** * Get the value for a given key as a {@link List}, e.g. where the value is known to be a - * delimited collection of items. If the value only contains one item, then a singleton - * {@link List} is returned. + * delimited collection of items (delimited by {@link AttributeMapUtil#VALUE_DELIMITER}). + * If the value only contains one item, then a singleton {@link List} is returned. */ - public List getAsList(final String key) { + public List getValueAsList(final CIKey key) { final String val = get(key); if (NullSafe.isEmptyString(val)) { return Collections.emptyList(); @@ -209,7 +438,21 @@ public List getAsList(final String key) { } } - public boolean isDelimited(final String key) { + /** + * Get a value from the {@link AttributeMap}, mapping it to a T if it is non-null. + * + * @return The value mapped to type T if key exists and has a non-null value, else null. + */ + public T getAs(final CIKey key, Function mapper) { + Objects.requireNonNull(mapper); + return GwtNullSafe.get(get(key), mapper); + } + +// public boolean isDelimited(final String key) { +// return isDelimited(CIKey.of(key)); +// } + + public boolean isDelimited(final CIKey key) { final String val = get(key); return NullSafe.test(val, val2 -> val2.contains(AttributeMapUtil.VALUE_DELIMITER)); @@ -218,7 +461,7 @@ public boolean isDelimited(final String key) { public static Builder copy(final AttributeMap copy) { final Builder builder = new Builder(); builder.overrideEmbeddedMeta = copy.overrideEmbeddedMeta; - builder.attributes = new AttributeMap(); + builder.attributes = new AttributeMap(copy); return builder; } @@ -226,11 +469,45 @@ public static Builder builder() { return new Builder(); } - public void removeAll(final Collection keySet) { +// public void removeAllStringKeys(final Collection keySet) { +// if (keySet != null) { +// for (final String key : keySet) { +// remove(CIKey.of(key)); +// } +// } +// } + + public void removeAll(final Collection keySet) { if (keySet != null) { - for (final String key : keySet) { - remove(key); - } + keySet.forEach(this::remove); + } + } + + /** + * @return A new {@link AttributeMap} that includes only the entries whose keys are in includeKeys + */ + public AttributeMap filterIncluding(final Set includeKeys) { + if (GwtNullSafe.isEmptyCollection(includeKeys)) { + return this; + } else { + return entrySet() + .stream() + .filter(entry -> includeKeys.contains(entry.getKey())) + .collect(collector()); + } + } + + /** + * @return A new {@link AttributeMap} that includes only the entries whose keys are not in excludeKeys + */ + public AttributeMap filterExcluding(final Set excludeKeys) { + if (GwtNullSafe.isEmptyCollection(excludeKeys)) { + return this; + } else { + return entrySet() + .stream() + .filter(entry -> !excludeKeys.contains(entry.getKey())) + .collect(collector()); } } @@ -238,6 +515,68 @@ public boolean isOverrideEmbeddedMeta() { return overrideEmbeddedMeta; } + /** + * @return A {@link Collector} for use with the {@link java.util.stream.Stream} API. + */ + public static Collector, AttributeMap, AttributeMap> collector() { + return AttributeMapCollector.INSTANCE; + } + + @Override + public String toString() { + return map.toString(); + } + + @Override + public boolean equals(final Object o) { + return map.equals(o); + } + + // -------------------------------------------------------------------------------- + + + /** + * {@link Collector} for use with {@link java.util.stream.Stream} API. + */ + public static class AttributeMapCollector + implements Collector, AttributeMap, AttributeMap> { + + private static final Collector, AttributeMap, AttributeMap> INSTANCE = + new AttributeMapCollector(); + + @Override + public Supplier supplier() { + return AttributeMap::new; + } + + @Override + public BiConsumer> accumulator() { + return (map, entry) -> { + if (entry != null) { + map.put(entry.getKey(), entry.getValue()); + } + }; + } + + @Override + public BinaryOperator combiner() { + return (map, map2) -> { + map.putAll(map2); + return map; + }; + } + + @Override + public Function finisher() { + return Function.identity(); + } + + @Override + public Set characteristics() { + return Set.of(Characteristics.UNORDERED, Characteristics.IDENTITY_FINISH); + } + } + // -------------------------------------------------------------------------------- @@ -264,13 +603,29 @@ public Builder overrideEmbeddedMeta() { return this; } + /** + * You really ought to be using a {@link CIKey} key. + */ public Builder put(final String key, final String value) { - Objects.requireNonNull(key); + this.put(CIKey.trimmed(key), value); + return this; + } + + public Builder put(final CIKey key, final String value) { attributes.put(key, value); return this; } + /** + * You really ought to be using a {@link CIKey} key. + */ + @Deprecated public Builder putCollection(final String key, Collection values) { + putCollection(CIKey.of(key), values); + return this; + } + + public Builder putCollection(final CIKey key, Collection values) { Objects.requireNonNull(key); attributes.putCollection(key, values); return this; diff --git a/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/AttributeMapUtil.java b/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/AttributeMapUtil.java index 3c56c844074..9d02a9bd117 100644 --- a/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/AttributeMapUtil.java +++ b/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/AttributeMapUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import stroom.util.NullSafe; import stroom.util.cert.CertificateExtractor; import stroom.util.io.StreamUtil; +import stroom.util.shared.string.CIKey; import jakarta.servlet.http.HttpServletRequest; import org.slf4j.Logger; @@ -141,7 +142,7 @@ public static void read(final String data, final AttributeMap attributeMap) { if (splitPos != -1) { final String key = line.substring(0, splitPos); String value = line.substring(splitPos + 1); - attributeMap.put(key.trim(), value.trim()); + attributeMap.put(key, value); } else { attributeMap.put(line, null); } @@ -158,7 +159,7 @@ public static void write(final AttributeMap attributeMap, final OutputStream out public static void appendAttributes(final AttributeMap attributeMap, final StringBuilder builder, - final String... attributeKeys) { + final CIKey... attributeKeys) { if (builder != null && attributeMap != null && attributeKeys != null) { @@ -190,7 +191,7 @@ public static List valueAsList(final String attributeValue) { } } - private static String getAttributeStr(final AttributeMap attributeMap, final String attributeKey) { + private static String getAttributeStr(final AttributeMap attributeMap, final CIKey attributeKey) { final String attributeValue = attributeMap.get(attributeKey); final String str; if (attributeValue != null && !attributeValue.isBlank()) { @@ -203,11 +204,12 @@ private static String getAttributeStr(final AttributeMap attributeMap, final Str public static void write(final AttributeMap attributeMap, final Writer writer) throws IOException { try { - attributeMap.entrySet().stream() - .sorted(Map.Entry.comparingByKey(String::compareToIgnoreCase)) + attributeMap.entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) .forEachOrdered(e -> { try { - writer.write(e.getKey()); + writer.write(e.getKey().get()); final String value = e.getValue(); if (value != null) { writer.write(HEADER_DELIMITER); diff --git a/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/CIStringHashMap.java b/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/CIStringHashMap.java deleted file mode 100644 index 0c5a4e865c7..00000000000 --- a/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/CIStringHashMap.java +++ /dev/null @@ -1,211 +0,0 @@ -package stroom.meta.api; - -import java.io.Serializable; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.function.Function; - -/** - * String hash map that does not care about key case. - */ -class CIStringHashMap implements Map { - - private final HashMap map = new HashMap<>(); - - @Override - public void clear() { - map.clear(); - } - - @Override - public boolean containsKey(final Object key) { - return map.containsKey(new CIString((String) key)); - } - - @Override - public boolean containsValue(final Object value) { - return map.containsValue(value); - } - - @Override - public String get(final Object key) { - return map.get(new CIString((String) key)); - } - - @Override - public String getOrDefault(Object key, String defaultVal) { - String val = map.get(new CIString((String) key)); - return val == null - ? defaultVal - : val; - } - - @Override - public boolean isEmpty() { - return map.isEmpty(); - } - - public String computeIfAbsent(final String key, final Function mappingFunction) { - return map.computeIfAbsent(new CIString(key), k -> mappingFunction.apply(k.key)); - } - - @Override - public String put(final String key, String value) { - if (value != null) { - value = value.trim(); - } - final CIString newKey = new CIString(key); - final String oldValue = map.remove(newKey); - map.put(newKey, value); - return oldValue; - } - - @Override - public String remove(final Object key) { - return map.remove(new CIString((String) key)); - } - - @Override - public int size() { - return map.size(); - } - - /** - * DOESN'T adhere to the contract of {@link Map#entrySet()}, so any changes - * to the returned {@link Set} will NOT affect this map. - */ - @Override - public Set> entrySet() { - final Set> rtnSet = new HashSet<>(); - for (final Entry entry : map.entrySet()) { - rtnSet.add(new CIEntryAdaptor(entry)); - } - return rtnSet; - } - - /** - * DOESN'T adhere to the contract of {@link Map#keySet()}, so any changes - * to the returned {@link Set} will NOT affect this map. - */ - @Override - public Set keySet() { - final Set rtnSet = new HashSet<>(); - for (final CIString entry : map.keySet()) { - rtnSet.add(entry.key); - } - return rtnSet; - } - - @Override - public void putAll(final Map m) { - for (final Entry entry : m.entrySet()) { - put(entry.getKey(), entry.getValue()); - } - } - - @Override - public Collection values() { - return map.values(); - } - - @Override - public String toString() { - return map.toString(); - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final CIStringHashMap that = (CIStringHashMap) o; - return Objects.equals(map, that.map); - } - - @Override - public int hashCode() { - return Objects.hash(map); - } - - - // -------------------------------------------------------------------------------- - - - protected static class CIString implements Comparable, Serializable { - - private final String key; - private final String lowerKey; - - CIString(final String key) { - this.key = key.trim(); - this.lowerKey = this.key.toLowerCase(Locale.ENGLISH); - } - - public String getKey() { - return key; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final CIString ciString = (CIString) o; - return lowerKey.equals(ciString.lowerKey); - } - - @Override - public int hashCode() { - return lowerKey.hashCode(); - } - - @Override - public int compareTo(final CIString o) { - return lowerKey.compareTo(o.lowerKey); - } - - @Override - public String toString() { - return key; - } - } - - - // -------------------------------------------------------------------------------- - - - private static class CIEntryAdaptor implements Entry { - - private final Entry realEntry; - - private CIEntryAdaptor(final Entry realEntry) { - this.realEntry = realEntry; - } - - @Override - public String getKey() { - return realEntry.getKey().key; - } - - @Override - public String getValue() { - return realEntry.getValue(); - } - - @Override - public String setValue(final String value) { - return realEntry.setValue(value); - } - } -} diff --git a/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/StandardHeaderArguments.java b/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/StandardHeaderArguments.java index 70c19616f17..5c9e3186456 100644 --- a/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/StandardHeaderArguments.java +++ b/stroom-meta/stroom-meta-api/src/main/java/stroom/meta/api/StandardHeaderArguments.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,15 @@ package stroom.meta.api; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; + import java.util.Set; public interface StandardHeaderArguments { - String GUID = "GUID"; - String COMPRESSION = "Compression"; + CIKey GUID = CIKeys.GUID; + CIKey COMPRESSION = CIKeys.COMPRESSION; String COMPRESSION_ZIP = "ZIP"; String COMPRESSION_GZIP = "GZIP"; String COMPRESSION_NONE = "NONE"; @@ -31,58 +34,58 @@ public interface StandardHeaderArguments { COMPRESSION_ZIP, COMPRESSION_NONE); - String CONTENT_LENGTH = "content-length"; + CIKey CONTENT_LENGTH = CIKeys.CONTENT___LENGTH; - String CONTENT_ENCODING = "content-encoding"; + CIKey CONTENT_ENCODING = CIKeys.CONTENT___ENCODING; String CONTENT_ENCODING_GZIP = "gzip"; String CONTENT_ENCODING_DEFLATE = "deflate"; String CONTENT_ENCODING_BROTLI = "br"; String CONTENT_ENCODING_ZSTD = "zstd"; - String USER_AGENT = "user-agent"; + CIKey USER_AGENT = CIKeys.USER___AGENT; - String REMOTE_ADDRESS = "RemoteAddress"; - String REMOTE_HOST = "RemoteHost"; - String RECEIVED_TIME = "ReceivedTime"; + CIKey REMOTE_ADDRESS = CIKeys.REMOTE_ADDRESS; + CIKey REMOTE_HOST = CIKeys.REMOTE_HOST; + CIKey RECEIVED_TIME = CIKeys.RECEIVED_TIME; /** * A comma delimited list of ReceivedTime values, oldest first that includes the * ReceivedTime value as its last item. */ - String RECEIVED_TIME_HISTORY = "ReceivedTimeHistory"; - String RECEIVED_PATH = "ReceivedPath"; - String EFFECTIVE_TIME = "EffectiveTime"; - String REMOTE_DN = "RemoteDN"; - String REMOTE_CERT_EXPIRY = "RemoteCertExpiry"; - String REMOTE_FILE = "RemoteFile"; + CIKey RECEIVED_TIME_HISTORY = CIKeys.RECEIVED_TIME_HISTORY; + CIKey RECEIVED_PATH = CIKeys.RECEIVED_PATH; + CIKey EFFECTIVE_TIME = CIKeys.EFFECTIVE_TIME; + CIKey REMOTE_DN = CIKeys.REMOTE_DN; + CIKey REMOTE_CERT_EXPIRY = CIKeys.REMOTE_CERT_EXPIRY; + CIKey REMOTE_FILE = CIKeys.REMOTE_FILE; // The unique identifier of the user on the IDP - String UPLOAD_USER_ID = "UploadUserId"; + CIKey UPLOAD_USER_ID = CIKeys.UPLOAD_USER_ID; // Username of the user on the IDP, may not be unique - String UPLOAD_USERNAME = "UploadUsername"; + CIKey UPLOAD_USERNAME = CIKeys.UPLOAD_USERNAME; - String STREAM_SIZE = "StreamSize"; + CIKey STREAM_SIZE = CIKeys.STREAM_SIZE; String STROOM_STATUS = "Stroom-Status"; String STROOM_ERROR = "Stroom-Error"; - String FEED = "Feed"; - String TYPE = "Type"; + CIKey FEED = CIKeys.FEED; + CIKey TYPE = CIKeys.TYPE; // Typically added in by nginx - String X_FORWARDED_FOR = "X-Forwarded-For"; + CIKey X_FORWARDED_FOR = CIKeys.X___FORWARDED___FOR; - Set HEADER_CLONE_EXCLUDE_SET = Set.of( - "accept", - "connection", - "content-length", - "transfer-encoding", - "expect", - COMPRESSION); + Set HEADER_CLONE_EXCLUDE_SET = Set.of( + CIKeys.ACCEPT, + CIKeys.CONNECTION, + CIKeys.CONTENT___LENGTH, + CIKeys.TRANSFER___ENCODING, + CIKeys.EXPECT, + CIKeys.COMPRESSION); /** * Header keys for values that are date/time strings */ - Set DATE_HEADER_KEYS = Set.of( - EFFECTIVE_TIME, - RECEIVED_TIME); + Set DATE_HEADER_KEYS = Set.of( + CIKeys.EFFECTIVE_TIME, + CIKeys.RECEIVED_TIME); } diff --git a/stroom-meta/stroom-meta-api/src/test/java/stroom/meta/api/TestAttributeMap.java b/stroom-meta/stroom-meta-api/src/test/java/stroom/meta/api/TestAttributeMap.java index 05bcd9216b9..f75877eb45d 100644 --- a/stroom-meta/stroom-meta-api/src/test/java/stroom/meta/api/TestAttributeMap.java +++ b/stroom-meta/stroom-meta-api/src/test/java/stroom/meta/api/TestAttributeMap.java @@ -1,8 +1,26 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.meta.api; import stroom.test.common.TestUtil; import stroom.util.NullSafe; import stroom.util.date.DateUtil; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import io.vavr.Tuple; import org.junit.jupiter.api.DynamicTest; @@ -27,6 +45,9 @@ class TestAttributeMap { + public static final CIKey FOO = CIKey.of("foo"); + public static final CIKey BAR = CIKey.of("bar"); + @Test void testSimple() { AttributeMap attributeMap = new AttributeMap(); @@ -35,14 +56,16 @@ void testSimple() { assertThat(attributeMap.get("person")).isEqualTo("person1"); assertThat(attributeMap.get("PERSON")).isEqualTo("person1"); - assertThat(attributeMap.keySet()).isEqualTo(new HashSet<>(Collections.singletonList("person"))); + assertThat(attributeMap.keySetAsStrings()) + .isEqualTo(new HashSet<>(Collections.singletonList("person"))); attributeMap.put("PERSON", "person2"); assertThat(attributeMap.get("person")).isEqualTo("person2"); assertThat(attributeMap.get("PERSON")).isEqualTo("person2"); - assertThat(attributeMap.keySet()).isEqualTo(new HashSet<>(Collections.singletonList("PERSON"))); + assertThat(attributeMap.keySetAsStrings()) + .isEqualTo(new HashSet<>(Collections.singletonList("person"))); AttributeMap attributeMap2 = new AttributeMap(); attributeMap2.put("persOn", "person3"); @@ -50,7 +73,8 @@ void testSimple() { attributeMap.putAll(attributeMap2); - assertThat(attributeMap.keySet()).isEqualTo(new HashSet<>(Arrays.asList("persOn", "persOn1"))); + assertThat(attributeMap.keySetAsStrings()) + .isEqualTo(new HashSet<>(Arrays.asList("person", "persOn1"))); } @Test @@ -59,9 +83,30 @@ void testRemove() { attributeMap.put("a", "a1"); attributeMap.put("B", "b1"); - attributeMap.removeAll(Arrays.asList("A", "b")); + attributeMap.remove(CIKey.of("A")); + + assertThat(attributeMap.size()) + .isEqualTo(1); + + attributeMap.remove(" b "); + + assertThat(attributeMap.size()) + .isEqualTo(0); + } + + @Test + void testRemoveAll() { + AttributeMap attributeMap = new AttributeMap(); + attributeMap.put("a", "a1"); + attributeMap.put("B", "b1"); + attributeMap.put("c", "c1"); + + attributeMap.removeAll(CIKey.listOf("A", "b")); - assertThat(attributeMap.size()).isEqualTo(0); + assertThat(attributeMap.size()) + .isEqualTo(1); + assertThat(attributeMap.keySet()) + .contains(CIKey.of("c")); } @Test @@ -72,8 +117,8 @@ void testReadWrite() throws IOException { assertThat(attributeMap.get("b")).isEqualTo("2"); assertThat(attributeMap.get("z")).isNull(); - assertThat(new String(AttributeMapUtil.toByteArray(attributeMap), AttributeMapUtil.DEFAULT_CHARSET)).isEqualTo( - "a:1\nb:2\nz\n"); + assertThat(new String(AttributeMapUtil.toByteArray(attributeMap), AttributeMapUtil.DEFAULT_CHARSET)) + .isEqualTo("a:1\nb:2\nz\n"); } @Test @@ -99,21 +144,22 @@ void testTrim() { assertThat(attributeMap.get("PERSON ")).isEqualTo("person2"); assertThat(attributeMap.get("FOOBAR")).isEqualTo("3"); + attributeMap.get("sss"); } @Test void testWriteMultiLineValues() throws IOException { AttributeMap attributeMap = AttributeMap.builder() - .put("foo", "123") - .put("files", "/some/path/file1,/some/path/file2,/some/path/file3") - .put("bar", "456") + .put(FOO, "123") + .put(CIKeys.FILES, "/some/path/file1,/some/path/file2,/some/path/file3") + .put(BAR, "456") .build(); final String str = new String(AttributeMapUtil.toByteArray(attributeMap), AttributeMapUtil.DEFAULT_CHARSET); assertThat(str) .isEqualTo(""" bar:456 - files:/some/path/file1,/some/path/file2,/some/path/file3 + Files:/some/path/file1,/some/path/file2,/some/path/file3 foo:123 """); } @@ -121,19 +167,19 @@ void testWriteMultiLineValues() throws IOException { @Test void testPutCollection() throws IOException { AttributeMap attributeMap = AttributeMap.builder() - .put("foo", "123") - .putCollection("files", List.of( + .put(FOO, "123") + .putCollection(CIKeys.FILES, List.of( "/some/path/file1", "/some/path/file2", "/some/path/file3")) - .put("bar", "456") + .put(BAR, "456") .build(); final String str = new String(AttributeMapUtil.toByteArray(attributeMap), AttributeMapUtil.DEFAULT_CHARSET); assertThat(str) .isEqualTo(""" bar:456 - files:/some/path/file1,/some/path/file2,/some/path/file3 + Files:/some/path/file1,/some/path/file2,/some/path/file3 foo:123 """); } @@ -141,22 +187,22 @@ void testPutCollection() throws IOException { @Test void testGetAsCollection() throws IOException { AttributeMap attributeMap = AttributeMap.builder() - .put("foo", "123") - .putCollection("files", List.of( + .put(FOO, "123") + .putCollection(CIKeys.FILES, List.of( "/some/path/file1", "/some/path/file2", "/some/path/file3")) - .put("bar", "456") + .put(BAR, "456") .build(); - assertThat(attributeMap.get("files")) + assertThat(attributeMap.get(CIKeys.FILES)) .isEqualTo("/some/path/file1,/some/path/file2,/some/path/file3"); - assertThat(attributeMap.getAsList("files")) + assertThat(attributeMap.getValueAsList(CIKeys.FILES)) .containsExactly( "/some/path/file1", "/some/path/file2", "/some/path/file3"); - assertThat(attributeMap.getAsList("foo")) + assertThat(attributeMap.getValueAsList(FOO)) .containsExactly("123"); } @@ -205,12 +251,12 @@ public synchronized int read(final byte[] b, final int off, final int len) { void testEquality1() { final AttributeMap attributeMap1 = new AttributeMap(); attributeMap1.putAll(Map.of( - "foo", "123", - "bar", "456")); + FOO, "123", + BAR, "456")); final AttributeMap attributeMap2 = new AttributeMap(); attributeMap2.putAll(Map.of( - "FOO", "123", - "BAR", "456")); + FOO, "123", + BAR, "456")); assertThat(attributeMap1) .isEqualTo(attributeMap2); @@ -219,12 +265,12 @@ void testEquality1() { @Test void testEquality2() { final AttributeMap attributeMap1 = new AttributeMap(); - attributeMap1.putAll(Map.of( + attributeMap1.putAll(CIKey.mapOf( "fooXXX", "123", "bar", "456")); final AttributeMap attributeMap2 = new AttributeMap(); - attributeMap2.putAll(Map.of( - "FOO", "123", + attributeMap2.putAll(CIKey.mapOf( + "foo", "123", "BAR", "456")); assertThat(attributeMap1) @@ -235,12 +281,12 @@ void testEquality2() { void testEquality3() { final AttributeMap attributeMap1 = new AttributeMap(); attributeMap1.putAll(Map.of( - "foo", "value1", - "bar", "value2")); + FOO, "value1", + BAR, "value2")); final AttributeMap attributeMap2 = new AttributeMap(); attributeMap2.putAll(Map.of( - "foo", "VALUE1", - "bar", "VALUE2")); + FOO, "VALUE1", + BAR, "VALUE2")); // Value cases not same assertThat(attributeMap1) @@ -251,8 +297,8 @@ void testEquality3() { Stream testGet() { final AttributeMap attributeMap1 = new AttributeMap(); attributeMap1.putAll(Map.of( - "foo", "123", - "bar", "456")); + FOO, "123", + BAR, "456")); final AttributeMap attributeMapEmpty = new AttributeMap(); return TestUtil.buildDynamicTestStream() @@ -263,7 +309,7 @@ Stream testGet() { return attrMap.get(testCase.getInput()._2); }) .withSimpleEqualityAssertion() - .addThrowsCase(Tuple.of(attributeMapEmpty, null), NullPointerException.class) + .addThrowsCase(Tuple.of(attributeMapEmpty, null), null) .addCase(Tuple.of(attributeMapEmpty, "foo"), null) .addCase(Tuple.of(attributeMap1, ""), null) .addCase(Tuple.of(attributeMap1, "foo"), "123") @@ -275,10 +321,38 @@ Stream testGet() { .build(); } + @TestFactory + Stream testGet2() { + final AttributeMap attributeMap1 = AttributeMap.builder() + .put(FOO, "123") + .put(BAR, "456") + .build(); + final AttributeMap attributeMapEmpty = new AttributeMap(); + + return TestUtil.buildDynamicTestStream() + .withInputTypes(AttributeMap.class, CIKey.class) + .withOutputType(String.class) + .withTestFunction(testCase -> { + var attrMap = testCase.getInput()._1; + return attrMap.get(testCase.getInput()._2); + }) + .withSimpleEqualityAssertion() + .addThrowsCase(Tuple.of(attributeMapEmpty, null), null) + .addCase(Tuple.of(attributeMapEmpty, CIKey.of("foo")), null) + .addCase(Tuple.of(attributeMap1, CIKey.of("")), null) + .addCase(Tuple.of(attributeMap1, CIKey.of("foo")), "123") + .addCase(Tuple.of(attributeMap1, CIKey.of("FOO")), "123") + .addCase(Tuple.of(attributeMap1, CIKey.of("Foo")), "123") + .addCase(Tuple.of(attributeMap1, CIKey.trimmed(" Foo")), "123") + .addCase(Tuple.of(attributeMap1, CIKey.trimmed("Foo ")), "123") + .addCase(Tuple.of(attributeMap1, CIKey.trimmed(" Foo ")), "123") + .build(); + } + @TestFactory Stream testContainsKey() { final AttributeMap attributeMap1 = new AttributeMap(); - attributeMap1.putAll(Map.of( + attributeMap1.putAll(CIKey.mapOf( "foo", "123", "bar", "456")); final AttributeMap attributeMapEmpty = new AttributeMap(); @@ -291,7 +365,7 @@ Stream testContainsKey() { return attrMap.containsKey(testCase.getInput()._2); }) .withSimpleEqualityAssertion() - .addThrowsCase(Tuple.of(attributeMapEmpty, null), NullPointerException.class) + .addCase(Tuple.of(attributeMapEmpty, null), false) .addCase(Tuple.of(attributeMapEmpty, "foo"), false) .addCase(Tuple.of(attributeMap1, ""), false) .addCase(Tuple.of(attributeMap1, "foo"), true) @@ -303,12 +377,40 @@ Stream testContainsKey() { .build(); } + @TestFactory + Stream testContainsKey2() { + final AttributeMap attributeMap1 = new AttributeMap(); + attributeMap1.putAll(CIKey.mapOf( + "foo", "123", + "bar", "456")); + final AttributeMap attributeMapEmpty = new AttributeMap(); + + return TestUtil.buildDynamicTestStream() + .withInputTypes(AttributeMap.class, CIKey.class) + .withOutputType(boolean.class) + .withTestFunction(testCase -> { + var attrMap = testCase.getInput()._1; + return attrMap.containsKey(testCase.getInput()._2); + }) + .withSimpleEqualityAssertion() + .addCase(Tuple.of(attributeMapEmpty, null), false) + .addCase(Tuple.of(attributeMapEmpty, CIKey.of("foo")), false) + .addCase(Tuple.of(attributeMap1, CIKey.of("")), false) + .addCase(Tuple.of(attributeMap1, CIKey.of("foo")), true) + .addCase(Tuple.of(attributeMap1, CIKey.of("FOO")), true) + .addCase(Tuple.of(attributeMap1, CIKey.of("Foo")), true) + .addCase(Tuple.of(attributeMap1, CIKey.trimmed(" Foo")), true) + .addCase(Tuple.of(attributeMap1, CIKey.trimmed("Foo ")), true) + .addCase(Tuple.of(attributeMap1, CIKey.trimmed(" Foo ")), true) + .build(); + } + @TestFactory Stream testContainsValue() { final AttributeMap attributeMap1 = new AttributeMap(); attributeMap1.putAll(Map.of( - "foo", "value1", - "bar", "value2")); + FOO, "value1", + BAR, "value2")); attributeMap1.put("NULL", null); final AttributeMap attributeMapEmpty = new AttributeMap(); @@ -334,24 +436,40 @@ Stream testContainsValue() { void testPut() { final AttributeMap attributeMap1 = new AttributeMap(); - assertThat(attributeMap1) - .isEmpty(); + assertThat(attributeMap1.isEmpty()) + .isTrue(); attributeMap1.put("foo", "value1a"); - assertThat(attributeMap1) - .hasSize(1); + assertThat(attributeMap1.size()) + .isEqualTo(1); assertThat(attributeMap1.get("Foo")) .isEqualTo("value1a"); + assertThat(attributeMap1.get(CIKey.of("Foo"))) + .isEqualTo("value1a"); attributeMap1.put("FOO", "value1b"); // 'same' key, new val - assertThat(attributeMap1) - .hasSize(1); + assertThat(attributeMap1.size()) + .isEqualTo(1); assertThat(attributeMap1.get("Foo")) .isEqualTo("value1b"); + attributeMap1.put(CIKey.of("FoO"), "value1c"); // 'same' key, new val + assertThat(attributeMap1.size()) + .isEqualTo(1); + assertThat(attributeMap1.get(CIKey.of("fOO"))) + .isEqualTo("value1c"); + + attributeMap1.put(" foo ", " value1d "); // Same after trimming + assertThat(attributeMap1.size()) + .isEqualTo(1); + assertThat(attributeMap1.get("Foo")) + .isEqualTo("value1d"); + assertThat(attributeMap1.get(CIKey.of("Foo"))) + .isEqualTo("value1d"); + attributeMap1.put("bar", "value2a"); - assertThat(attributeMap1) - .hasSize(2); + assertThat(attributeMap1.size()) + .isEqualTo(2); assertThat(attributeMap1.get("BAR")) .isEqualTo("value2a"); } @@ -362,15 +480,15 @@ void testPut_withDateNormalisation() { final String dateStrIn = "2010-01-01T23:59:59.123456+00:00"; final String dateStrOut = "2010-01-01T23:59:59.123Z"; - for (final String key : StandardHeaderArguments.DATE_HEADER_KEYS) { + for (final CIKey key : StandardHeaderArguments.DATE_HEADER_KEYS) { attributeMap1.clear(); - assertThat(attributeMap1) - .isEmpty(); + assertThat(attributeMap1.isEmpty()) + .isTrue(); attributeMap1.put(key, dateStrIn); - assertThat(attributeMap1) - .hasSize(1); + assertThat(attributeMap1.size()) + .isEqualTo(1); assertThat(attributeMap1.get(key)) .isEqualTo(dateStrOut); @@ -383,15 +501,15 @@ void testPutDateTime1() { final String dateStrIn = "2010-01-01T23:59:59.123456+00:00"; final String dateStrOut = "2010-01-01T23:59:59.123Z"; final long epochMs = Instant.parse(dateStrIn).toEpochMilli(); - final String key = "foo"; + final CIKey key = FOO; - assertThat(attributeMap1) - .isEmpty(); + assertThat(attributeMap1.isEmpty()) + .isTrue(); attributeMap1.putDateTime(key, epochMs); - assertThat(attributeMap1) - .hasSize(1); + assertThat(attributeMap1.size()) + .isEqualTo(1); assertThat(attributeMap1.get(key)) .isEqualTo(dateStrOut); @@ -403,15 +521,15 @@ void testPutDateTime2() { final String dateStrIn = "2010-01-01T23:59:59.123456+00:00"; final String dateStrOut = "2010-01-01T23:59:59.123Z"; final Instant instant = Instant.parse(dateStrIn); - final String key = "foo"; + final CIKey key = FOO; - assertThat(attributeMap1) - .isEmpty(); + assertThat(attributeMap1.isEmpty()) + .isTrue(); attributeMap1.putDateTime(key, instant); - assertThat(attributeMap1) - .hasSize(1); + assertThat(attributeMap1.size()) + .isEqualTo(1); assertThat(attributeMap1.get(key)) .isEqualTo(dateStrOut); @@ -422,16 +540,16 @@ void testPutCurrentDateTime() { final AttributeMap attributeMap1 = new AttributeMap(); final String dateStrIn = "2010-01-01T23:59:59.123456+00:00"; final String dateStrOut = "2010-01-01T23:59:59.123Z"; - final String key = "foo"; + final CIKey key = FOO; - assertThat(attributeMap1) - .isEmpty(); + assertThat(attributeMap1.isEmpty()) + .isTrue(); final Instant now = Instant.now(); attributeMap1.putCurrentDateTime(key); - assertThat(attributeMap1) - .hasSize(1); + assertThat(attributeMap1.size()) + .isEqualTo(1); final String val = attributeMap1.get(key); assertThat(val) @@ -443,7 +561,7 @@ void testPutCurrentDateTime() { @Test void testAppendDateTime_notPresent() { - final String key = "foo"; + final CIKey key = FOO; final Instant instant1 = Instant.now(); final String str1 = DateUtil.createNormalDateTimeString(instant1); @@ -458,7 +576,7 @@ void testAppendDateTime_notPresent() { @Test void testAppendDateTime_present() { - final String key = "foo"; + final CIKey key = FOO; final Instant instant1 = Instant.now().minus(1, ChronoUnit.DAYS); final String str1 = DateUtil.createNormalDateTimeString(instant1); final Instant instant2 = Instant.now(); @@ -482,7 +600,7 @@ void testAppendDateTime_present() { @Test void testAppendDateTime_present_null() { - final String key = "foo"; + final CIKey key = FOO; final Instant instant1 = Instant.now(); final String str1 = DateUtil.createNormalDateTimeString(instant1); @@ -498,7 +616,7 @@ void testAppendDateTime_present_null() { @Test void testAppendDateTime_present_emptyString() { - final String key = "foo"; + final CIKey key = FOO; final Instant instant1 = Instant.now(); final String str1 = DateUtil.createNormalDateTimeString(instant1); @@ -514,7 +632,7 @@ void testAppendDateTime_present_emptyString() { @Test void testAppendItem_notPresent() { - final String key = "foo"; + final CIKey key = FOO; final String item1 = "1"; final AttributeMap attributeMap = new AttributeMap(); @@ -528,7 +646,7 @@ void testAppendItem_notPresent() { @Test void testAppendItem_present() { - final String key = "foo"; + final CIKey key = FOO; final String item1 = "1"; final String item2 = "2"; @@ -550,7 +668,7 @@ void testAppendItem_present() { @Test void testAppendItem_present_null() { - final String key = "foo"; + final CIKey key = FOO; final String item1 = "1"; final AttributeMap attributeMap = new AttributeMap(); @@ -565,7 +683,7 @@ void testAppendItem_present_null() { @Test void testAppendItem_present_emptyString() { - final String key = "foo"; + final CIKey key = FOO; final String item1 = "1"; final AttributeMap attributeMap = new AttributeMap(); @@ -580,7 +698,7 @@ void testAppendItem_present_emptyString() { @Test void testAppendItemIf_notPresent() { - final String key = "foo"; + final CIKey key = FOO; final String item1 = "1"; final AttributeMap attributeMap = new AttributeMap(); @@ -606,7 +724,7 @@ void testAppendItemIf_notPresent() { @Test void testAppendItemIf_present() { - final String key = "foo"; + final CIKey key = FOO; final String item1 = "1"; final String item2 = "2"; @@ -634,11 +752,11 @@ void testAppendItemIf_present() { void testComputeIfAbsent1() { final AttributeMap attributeMap1 = new AttributeMap(); - assertThat(attributeMap1) - .isEmpty(); + assertThat(attributeMap1.isEmpty()) + .isTrue(); final AtomicInteger callCount = new AtomicInteger(); - final String computedVal = attributeMap1.computeIfAbsent("foo", k -> { + final String computedVal = attributeMap1.computeIfAbsent(FOO, k -> { callCount.incrementAndGet(); return "value(" + k + ")"; }); @@ -654,11 +772,11 @@ void testComputeIfAbsent2() { final AttributeMap attributeMap1 = new AttributeMap(); attributeMap1.put("foo", "value(initial)"); - assertThat(attributeMap1) - .hasSize(1); + assertThat(attributeMap1.size()) + .isEqualTo(1); final AtomicInteger callCount = new AtomicInteger(); - final String computedVal = attributeMap1.computeIfAbsent("foo", k -> { + final String computedVal = attributeMap1.computeIfAbsent(FOO, k -> { callCount.incrementAndGet(); return "value(" + k + ")"; }); @@ -668,4 +786,77 @@ void testComputeIfAbsent2() { assertThat(callCount) .hasValue(0); } + + @Test + void testGetAs() { + AttributeMap attributeMap = new AttributeMap(); + long nowMs = Instant.now().toEpochMilli(); + attributeMap.putDateTime(FOO, nowMs); + + final Long nowMs2 = attributeMap.getAs(FOO, DateUtil::parseNormalDateTimeString); + + assertThat(nowMs2) + .isEqualTo(nowMs); + } + + @Test + void testFilterIncluding() { + final AttributeMap attributeMap = new AttributeMap(); + attributeMap.put("a", "1"); + attributeMap.put("B", "1"); + attributeMap.put("c", "1"); + attributeMap.put("D", "1"); + attributeMap.put("e", "1"); + + final AttributeMap map = attributeMap.filterIncluding(CIKey.setOf("b", "d")); + + assertThat(map.keySet()) + .extracting(CIKey::getAsLowerCase) + .containsExactlyInAnyOrder("b", "d"); + } + + @Test + void testFilterExcluding() { + final AttributeMap attributeMap = new AttributeMap(); + attributeMap.put("a", "1"); + attributeMap.put("B", "1"); + attributeMap.put("c", "1"); + attributeMap.put("D", "1"); + attributeMap.put("e", "1"); + + final AttributeMap map = attributeMap.filterExcluding(CIKey.setOf("b", "d")); + + assertThat(map.keySet()) + .extracting(CIKey::getAsLowerCase) + .containsExactlyInAnyOrder("a", "c", "e"); + } + + @Test + void testCollector() { + + final AttributeMap attributeMap = Stream.of("a", "B", "c", "D", "e") + .map(key -> Map.entry(CIKey.of(key), key)) + .collect(AttributeMap.collector()); + + assertThat(attributeMap.keySet()) + .containsExactlyInAnyOrder( + CIKey.of("a"), + CIKey.of("B"), + CIKey.of("c"), + CIKey.of("D"), + CIKey.of("e")); + + assertThat(attributeMap.values()) + .containsExactlyInAnyOrder("a", "B", "c", "D", "e"); + } + + @Test + void testKeySetAsString() { + final AttributeMap attributeMap = Stream.of("a", "B", "c", "D", "e") + .map(key -> Map.entry(CIKey.of(key), key)) + .collect(AttributeMap.collector()); + + assertThat(attributeMap.keySetAsStrings()) + .containsExactlyInAnyOrder("a", "B", "c", "D", "e"); + } } diff --git a/stroom-meta/stroom-meta-api/src/test/java/stroom/meta/api/TestAttributeMapUtil.java b/stroom-meta/stroom-meta-api/src/test/java/stroom/meta/api/TestAttributeMapUtil.java index 27fd2bba927..88c7ee3026f 100644 --- a/stroom-meta/stroom-meta-api/src/test/java/stroom/meta/api/TestAttributeMapUtil.java +++ b/stroom-meta/stroom-meta-api/src/test/java/stroom/meta/api/TestAttributeMapUtil.java @@ -1,8 +1,25 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.meta.api; import stroom.test.common.TestUtil; import stroom.util.NullSafe; import stroom.util.exception.ThrowingFunction; +import stroom.util.shared.string.CIKey; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.DynamicTest; @@ -15,7 +32,6 @@ import java.nio.charset.Charset; import java.time.LocalDateTime; import java.util.Locale; -import java.util.Map; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -51,13 +67,13 @@ void testDataFormatter2() { @TestFactory Stream testCreate_fromString() { - final AttributeMap attributeMap1 = new AttributeMap(Map.of( + final AttributeMap attributeMap1 = new AttributeMap(CIKey.mapOf( "foo", "123")); - final AttributeMap attributeMap2 = new AttributeMap(Map.of( + final AttributeMap attributeMap2 = new AttributeMap(CIKey.mapOf( "foo", "123", "bar", "456")); - final AttributeMap attributeMap3 = new AttributeMap(Map.of( + final AttributeMap attributeMap3 = new AttributeMap(CIKey.mapOf( "files", "/some/path/file1,/some/path/file2,/some/path/file3", "foo", "123")); @@ -108,10 +124,10 @@ Stream testCreate_fromString() { @TestFactory Stream testCreate_fromInputStream() { - final AttributeMap attributeMap1 = new AttributeMap(Map.of( + final AttributeMap attributeMap1 = new AttributeMap(CIKey.mapOf( "foo", "123")); - final AttributeMap attributeMap2 = new AttributeMap(Map.of( + final AttributeMap attributeMap2 = new AttributeMap(CIKey.mapOf( "foo", "123", "bar", "456")); diff --git a/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaDaoImpl.java b/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaDaoImpl.java index 3558b3777d0..c16823f0c79 100644 --- a/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaDaoImpl.java +++ b/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaDaoImpl.java @@ -74,6 +74,7 @@ import stroom.util.shared.PageRequest; import stroom.util.shared.Range; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import stroom.util.time.TimePeriod; import io.vavr.Tuple; @@ -104,7 +105,6 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -314,7 +314,11 @@ private List getTypeIds(final List wildCardedTypeNames) { private List getPipelineUuidsByName(final List pipelineNames) { // Can't cache this in a simple map due to pipes being renamed, but // docRefInfoService should cache most of this anyway. - return docRefInfoService.findByNames(PipelineDoc.DOCUMENT_TYPE, pipelineNames, true) + return docRefInfoService.findByNames( + PipelineDoc.DOCUMENT_TYPE, + pipelineNames, + true, + false) .stream() .map(DocRef::getUuid) .collect(Collectors.toList()); @@ -1229,25 +1233,43 @@ public int count(final FindMetaCriteria criteria) { return (Integer) result; } - private boolean isUsed(final Set fieldSet, - final String[] resultFields, + private boolean isUsed(final CIKey field, + final List resultFields, final ExpressionCriteria criteria) { - return Arrays.stream(resultFields).filter(Objects::nonNull).anyMatch(fieldSet::contains) || - ExpressionUtil.termCount(criteria.getExpression(), fieldSet) > 0; + final boolean isInResultFields = NullSafe.stream(resultFields) + .filter(Objects::nonNull) + .anyMatch(resultField -> Objects.equals(resultField, field)); + + return isInResultFields + || ExpressionUtil.termCount(criteria.getExpression(), field.get()) > 0; + } + + private boolean isUsed(final Set fieldSet, + final List resultFields, + final ExpressionCriteria criteria) { + final boolean isInResultFields = NullSafe.stream(resultFields) + .filter(Objects::nonNull) + .anyMatch(fieldSet::contains); + + return isInResultFields || ExpressionUtil.termCount( + criteria.getExpression(), + fieldSet.stream() + .map(CIKey::get).collect(Collectors.toSet())) > 0; } @Override public void search(final ExpressionCriteria criteria, final FieldIndex fieldIndex, final ValuesConsumer consumer) { - final String[] fieldNames = fieldIndex.getFields(); - final boolean feedUsed = isUsed(Set.of(MetaFields.FEED.getFldName()), fieldNames, criteria); - final boolean typeUsed = isUsed(Set.of(MetaFields.TYPE.getFldName()), fieldNames, criteria); - final boolean pipelineUsed = isUsed(Set.of(MetaFields.PIPELINE.getFldName()), fieldNames, criteria); - final Set extendedFieldNames = MetaFields + + final List fieldNames = fieldIndex.getFieldsAsCIKeys(); + final boolean feedUsed = isUsed(MetaFields.FEED.getFldNameAsCIKey(), fieldNames, criteria); + final boolean typeUsed = isUsed(MetaFields.TYPE.getFldNameAsCIKey(), fieldNames, criteria); + final boolean pipelineUsed = isUsed(MetaFields.PIPELINE.getFldNameAsCIKey(), fieldNames, criteria); + final Set extendedFieldNames = MetaFields .getExtendedFields() .stream() - .map(QueryField::getFldName) + .map(QueryField::getFldNameAsCIKey) .collect(Collectors.toSet()); final boolean extendedValuesUsed = isUsed(extendedFieldNames, fieldNames, criteria); @@ -1258,12 +1280,13 @@ public void search(final ExpressionCriteria criteria, final Mapper[] mappers = valueMapper.getMappersForFieldNames(fieldNames); // Deal with extended fields. - final int[] extendedFieldKeys = new int[fieldNames.length]; + final int fieldCount = fieldNames.size(); + final int[] extendedFieldKeys = new int[fieldCount]; final List extendedFieldKeyIdList = new ArrayList<>(); final Map> extendedFieldValueMap = new HashMap<>(); - for (int i = 0; i < fieldNames.length; i++) { + for (int i = 0; i < fieldCount; i++) { final int index = i; - final String fieldName = fieldNames[i]; + final CIKey fieldName = fieldNames.get(i); extendedFieldKeys[i] = -1; if (extendedFieldNames.contains(fieldName)) { @@ -1324,14 +1347,14 @@ public void search(final ExpressionCriteria criteria, } result.forEach(r -> { - final Val[] arr = new Val[fieldNames.length]; + final Val[] arr = new Val[fieldCount]; Map extendedValues = null; if (extendedValuesUsed) { extendedValues = extendedFieldValueMap.get(r.get(meta.ID)); } - for (int i = 0; i < fieldNames.length; i++) { + for (int i = 0; i < fieldCount; i++) { Val val = ValNull.INSTANCE; final Mapper mapper = mappers[i]; if (mapper != null) { @@ -1959,10 +1982,10 @@ public boolean validateExpressionTerms(final ExpressionItem expressionItem) { if (expressionItem == null) { return true; } else { - final Map fieldMap = MetaFields.getAllFieldMap(); + final Map fieldMap = MetaFields.getAllFieldMap(); return ExpressionUtil.validateExpressionTerms(expressionItem, term -> { - final QueryField field = fieldMap.get(term.getField()); + final QueryField field = fieldMap.get(MetaFields.createCIKey(term.getField())); if (field == null) { throw new RuntimeException(LogUtil.message("Unknown field {} in term {}, in expression {}", term.getField(), term, expressionItem)); diff --git a/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaKeyDaoImpl.java b/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaKeyDaoImpl.java index 61196e89537..5c41905c098 100644 --- a/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaKeyDaoImpl.java +++ b/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaKeyDaoImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import stroom.meta.impl.MetaKeyDao; import stroom.meta.shared.MetaFields; import stroom.util.shared.Clearable; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -70,7 +71,7 @@ class MetaKeyDaoImpl implements MetaKeyDao, Clearable { private final MetaDbConnProvider metaDbConnProvider; private final Map idToNameCache = new HashMap<>(); - private final Map nameToIdCache = new HashMap<>(); + private final Map nameToIdCache = new HashMap<>(); @Inject MetaKeyDaoImpl(final MetaDbConnProvider metaDbConnProvider) { @@ -84,7 +85,7 @@ public Optional getNameForId(final int keyId) { } @Override - public Optional getIdForName(final String name) { + public Optional getIdForName(final CIKey name) { return Optional.ofNullable(nameToIdCache.get(name)); } @@ -135,7 +136,7 @@ private void fillCache() { final Integer id = r.get(META_KEY.ID); final String name = r.get(META_KEY.NAME); idToNameCache.put(id, name); - nameToIdCache.put(name, id); + nameToIdCache.put(CIKey.of(name), id); }); } diff --git a/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaValueDaoImpl.java b/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaValueDaoImpl.java index ae89369cad4..b6ce5fcb058 100644 --- a/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaValueDaoImpl.java +++ b/stroom-meta/stroom-meta-impl-db/src/main/java/stroom/meta/impl/db/MetaValueDaoImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +29,7 @@ import stroom.util.logging.LogExecutionTime; import stroom.util.logging.LogUtil; import stroom.util.shared.Clearable; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -216,13 +217,13 @@ private long getAttributeCreateTimeThresholdEpochMs() { } /** - * Convert a basic data list to a list of meta data using data attribute keys and values. + * Convert a basic meta list to a list of meta data using data attribute keys and values. */ @Override - public Map> getAttributes(final List list) { - final Map> attributeMap = new HashMap<>(); + public Map> getAttributes(final List list) { + final Map> attributeMap = new HashMap<>(); - // Get a list of valid data ids. + // Get a list of valid meta ids. final List idList = list.stream() .map(Meta::getId) .collect(Collectors.toList()); @@ -239,9 +240,10 @@ public Map> getAttributes(final List list) { .forEach(r -> { final int keyId = r.get(META_VAL.META_KEY_ID); metaKeyService.getNameForId(keyId).ifPresent(name -> { - final long dataId = r.get(META_VAL.META_ID); + final long metaId = r.get(META_VAL.META_ID); final String value = String.valueOf(r.get(META_VAL.VAL)); - attributeMap.computeIfAbsent(dataId, k -> new HashMap<>()).put(name, value); + attributeMap.computeIfAbsent(metaId, k -> new HashMap<>()) + .put(CIKey.of(name), value); }); }); diff --git a/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaKeyDao.java b/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaKeyDao.java index cd2a5da2348..104ef7e83ad 100644 --- a/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaKeyDao.java +++ b/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaKeyDao.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,18 +12,23 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.meta.impl; +import stroom.util.shared.string.CIKey; + import java.util.Optional; public interface MetaKeyDao { Optional getNameForId(final int keyId); - Optional getIdForName(final String name); + Optional getIdForName(final CIKey name); + + default Optional getIdForName(final String name) { + return getIdForName(CIKey.of(name)); + } Integer getMinId(); diff --git a/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaServiceImpl.java b/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaServiceImpl.java index 416bfb1cf2d..04284f3d358 100644 --- a/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaServiceImpl.java +++ b/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaServiceImpl.java @@ -59,6 +59,7 @@ import stroom.util.shared.PageRequest; import stroom.util.shared.ResultPage; import stroom.util.shared.UserRef; +import stroom.util.shared.string.CIKey; import stroom.util.time.TimePeriod; import jakarta.inject.Inject; @@ -153,10 +154,8 @@ public Meta getMeta(final long id, final boolean anyStatus) { final FindMetaCriteria findMetaCriteria = new FindMetaCriteria(secureExpression); findMetaCriteria.setPageRequest(PageRequest.oneRow()); final List list = find(findMetaCriteria).getValues(); - if (list == null || list.size() == 0) { - return null; - } - return list.get(0); + return NullSafe.first(list) + .orElse(null); } @Override @@ -506,7 +505,9 @@ public ResultPage findDecoratedRows(final FindMetaCriteria criteria) { () -> { final StreamAttributeMapRetentionRuleDecorator decorator = decoratorProvider.get(); list.getValues().forEach(metaRow -> - decorator.addMatchingRetentionRuleInfo(metaRow.getMeta(), metaRow.getAttributes())); + decorator.addMatchingRetentionRuleInfo( + metaRow.getMeta(), + metaRow.getAttributes())); }, "Adding data retention rules"); @@ -556,13 +557,16 @@ private List decorate(final List metaList) { } LOGGER.debug("Loading attribute map from DB"); - final Map> attributeMap = metaValueDao.getAttributes(metaList); + final Map> attributeMaps = metaValueDao.getAttributes(metaList); final List metaRowList = new ArrayList<>(metaList.size()); for (final Meta meta : metaList) { - final Map attributes = attributeMap.getOrDefault( + final Map attributes = attributeMaps.getOrDefault( meta.getId(), new HashMap<>()); - metaRowList.add(new MetaRow(meta, getPipelineName(meta), attributes)); + metaRowList.add(new MetaRow( + meta, + getPipelineName(meta), + CIKey.convertToStringMap(attributes))); } return metaRowList; }, @@ -622,7 +626,7 @@ private void addParents(final Meta child, final boolean anyStatus, final List parents = find(new FindMetaCriteria(getIdExpression(child.getParentMetaId(), anyStatus))).getValues(); - if (GwtNullSafe.hasItems(parents)) { + if (NullSafe.hasItems(parents)) { parents.forEach(parent -> { result.add(parent); addParents(parent, anyStatus, result); diff --git a/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaValueDao.java b/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaValueDao.java index c347b2be8ae..9628f4d8a09 100644 --- a/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaValueDao.java +++ b/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/MetaValueDao.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import stroom.meta.api.AttributeMap; import stroom.meta.shared.Meta; import stroom.util.shared.Flushable; +import stroom.util.shared.string.CIKey; import java.util.Collection; import java.util.List; @@ -28,7 +29,7 @@ public interface MetaValueDao extends Flushable { void addAttributes(Meta meta, AttributeMap attributes); - Map> getAttributes(List list); + Map> getAttributes(List list); void deleteOldValues(); diff --git a/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/StreamAttributeMapRetentionRuleDecorator.java b/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/StreamAttributeMapRetentionRuleDecorator.java index c16c0bf624f..aff09b96a5b 100644 --- a/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/StreamAttributeMapRetentionRuleDecorator.java +++ b/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/StreamAttributeMapRetentionRuleDecorator.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,19 +12,21 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.meta.impl; import stroom.data.retention.shared.DataRetentionRule; import stroom.data.retention.shared.DataRetentionRules; +import stroom.datasource.api.v2.QueryField; import stroom.expression.matcher.ExpressionMatcher; import stroom.expression.matcher.ExpressionMatcherFactory; import stroom.meta.shared.DataRetentionFields; import stroom.meta.shared.Meta; import stroom.meta.shared.MetaFields; +import stroom.util.NullSafe; import stroom.util.date.DateUtil; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -57,20 +59,21 @@ public StreamAttributeMapRetentionRuleDecorator(final ExpressionMatcherFactory e .orElse(Collections.emptyList()); } - void addMatchingRetentionRuleInfo(final Meta meta, final Map attributeMap) { + void addMatchingRetentionRuleInfo(final Meta meta, + final Map attributeMap) { try { int index = -1; // If there are no active rules then we aren't going to process anything. - if (rules != null && rules.size() > 0) { + if (NullSafe.hasItems(rules)) { // Create an attribute map we can match on. - final Map map = StreamAttributeMapUtil.createAttributeMap(meta, attributeMap); + final Map map = StreamAttributeMapUtil.createAttributeMap(meta, attributeMap); index = findMatchingRuleIndex(map); } if (index != -1) { final DataRetentionRule rule = rules.get(index); - attributeMap.put(DataRetentionFields.RETENTION_AGE, rule.getAgeString()); + putAttr(attributeMap, DataRetentionFields.RETENTION_AGE_FIELD, rule.getAgeString()); String keepUntil = DataRetentionRule.FOREVER; if (meta != null) { @@ -84,21 +87,27 @@ void addMatchingRetentionRuleInfo(final Meta meta, final Map att } } - attributeMap.put(DataRetentionFields.RETENTION_UNTIL, keepUntil); - attributeMap.put(DataRetentionFields.RETENTION_RULE, rule.toString()); + putAttr(attributeMap, DataRetentionFields.RETENTION_UNTIL_FIELD, keepUntil); + putAttr(attributeMap, DataRetentionFields.RETENTION_RULE_FIELD, rule.toString()); } else { - attributeMap.put(DataRetentionFields.RETENTION_AGE, DataRetentionRule.FOREVER); - attributeMap.put(DataRetentionFields.RETENTION_UNTIL, DataRetentionRule.FOREVER); - attributeMap.put(DataRetentionFields.RETENTION_RULE, "None"); + putAttr(attributeMap, DataRetentionFields.RETENTION_AGE_FIELD, DataRetentionRule.FOREVER); + putAttr(attributeMap, DataRetentionFields.RETENTION_UNTIL_FIELD, DataRetentionRule.FOREVER); + putAttr(attributeMap, DataRetentionFields.RETENTION_RULE_FIELD, "None"); } } catch (final RuntimeException e) { - attributeMap.put(DataRetentionFields.RETENTION_AGE, DataRetentionRule.FOREVER); - attributeMap.put(DataRetentionFields.RETENTION_UNTIL, DataRetentionRule.FOREVER); - attributeMap.put(DataRetentionFields.RETENTION_RULE, "Error - " + e.getMessage()); + putAttr(attributeMap, DataRetentionFields.RETENTION_AGE_FIELD, DataRetentionRule.FOREVER); + putAttr(attributeMap, DataRetentionFields.RETENTION_UNTIL_FIELD, DataRetentionRule.FOREVER); + putAttr(attributeMap, DataRetentionFields.RETENTION_RULE_FIELD, "Error - " + e.getMessage()); } } - private int findMatchingRuleIndex(final Map attributeMap) { + private void putAttr(final Map attributeMap, + final QueryField queryField, + final String value) { + attributeMap.put(queryField.getFldName(), value); + } + + private int findMatchingRuleIndex(final Map attributeMap) { RuntimeException lastException = null; for (int i = 0; i < rules.size(); i++) { diff --git a/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/StreamAttributeMapUtil.java b/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/StreamAttributeMapUtil.java index 54fada4f771..87aa90ea918 100644 --- a/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/StreamAttributeMapUtil.java +++ b/stroom-meta/stroom-meta-impl/src/main/java/stroom/meta/impl/StreamAttributeMapUtil.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.meta.impl; import stroom.docref.DocRef; @@ -6,6 +22,7 @@ import stroom.meta.shared.Status; import stroom.util.NullSafe; import stroom.util.date.DateUtil; +import stroom.util.shared.string.CIKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,28 +41,29 @@ private StreamAttributeMapUtil() { /** * Turns a stream attribute map object into a generic map of attributes for use by an expression filter. */ - static Map createAttributeMap(final Meta meta, final Map attributeMap) { - final Map map = new HashMap<>(); + static Map createAttributeMap(final Meta meta, + final Map attributeMap) { + final Map map = new HashMap<>(); if (meta != null) { - map.put(MetaFields.ID.getFldName(), meta.getId()); - map.put(MetaFields.CREATE_TIME.getFldName(), meta.getCreateMs()); - map.put(MetaFields.EFFECTIVE_TIME.getFldName(), meta.getEffectiveMs()); - map.put(MetaFields.STATUS_TIME.getFldName(), meta.getStatusMs()); - map.put(MetaFields.STATUS.getFldName(), NullSafe.get(meta.getStatus(), Status::getDisplayValue)); + map.put(MetaFields.ID.getFldNameAsCIKey(), meta.getId()); + map.put(MetaFields.CREATE_TIME.getFldNameAsCIKey(), meta.getCreateMs()); + map.put(MetaFields.EFFECTIVE_TIME.getFldNameAsCIKey(), meta.getEffectiveMs()); + map.put(MetaFields.STATUS_TIME.getFldNameAsCIKey(), meta.getStatusMs()); + map.put(MetaFields.STATUS.getFldNameAsCIKey(), NullSafe.get(meta.getStatus(), Status::getDisplayValue)); if (meta.getParentMetaId() != null) { - map.put(MetaFields.PARENT_ID.getFldName(), meta.getParentMetaId()); + map.put(MetaFields.PARENT_ID.getFldNameAsCIKey(), meta.getParentMetaId()); } if (meta.getTypeName() != null) { - map.put(MetaFields.TYPE.getFldName(), meta.getTypeName()); + map.put(MetaFields.TYPE.getFldNameAsCIKey(), meta.getTypeName()); } final String feedName = meta.getFeedName(); if (feedName != null) { - map.put(MetaFields.FEED.getFldName(), feedName); + map.put(MetaFields.FEED.getFldNameAsCIKey(), feedName); } final String pipelineUuid = meta.getPipelineUuid(); if (pipelineUuid != null) { - map.put(MetaFields.PIPELINE.getFldName(), new DocRef("Pipeline", pipelineUuid)); + map.put(MetaFields.PIPELINE.getFldNameAsCIKey(), new DocRef("Pipeline", pipelineUuid)); } // if (streamProcessor != null) { // final String pipelineUuid = streamProcessor.getPipelineUuid(); @@ -56,23 +74,14 @@ static Map createAttributeMap(final Meta meta, final Map { - final String value = attributeMap.get(field.getFldName()); + final CIKey fieldKey = field.getFldNameAsCIKey(); + final String value = attributeMap.get(fieldKey.get()); if (value != null) { try { switch (field.getFldType()) { - case TEXT: - map.put(field.getFldName(), value); - break; - case DATE: - map.put(field.getFldName(), DateUtil.parseNormalDateTimeString(value)); - break; - case DOC_REF: - attributeMap.put(field.getFldName(), value); - break; - case ID: - case LONG: - map.put(field.getFldName(), Long.valueOf(value)); - break; + case TEXT, DOC_REF -> map.put(fieldKey, value); + case DATE -> map.put(fieldKey, DateUtil.parseNormalDateTimeString(value)); + case ID, LONG -> map.put(fieldKey, Long.valueOf(value)); } } catch (final RuntimeException e) { LOGGER.error(e.getMessage(), e); diff --git a/stroom-meta/stroom-meta-mock/src/main/java/stroom/meta/mock/MockMetaService.java b/stroom-meta/stroom-meta-mock/src/main/java/stroom/meta/mock/MockMetaService.java index 82255ab08b9..72313ff27c7 100644 --- a/stroom-meta/stroom-meta-mock/src/main/java/stroom/meta/mock/MockMetaService.java +++ b/stroom-meta/stroom-meta-mock/src/main/java/stroom/meta/mock/MockMetaService.java @@ -40,6 +40,7 @@ import stroom.util.NullSafe; import stroom.util.shared.Clearable; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import stroom.util.time.TimePeriod; import jakarta.inject.Singleton; @@ -205,9 +206,9 @@ public ResultPage find(final FindMetaCriteria criteria) { try { final Meta meta = entry.getValue(); // final MetaRow row = new MetaRow(meta); - final Map attributeMap = createAttributeMap(meta); - if (criteria.getExpression() == null || expressionMatcher.match(attributeMap, - criteria.getExpression())) { + final Map attributeMap = createAttributeMap(meta); + if (criteria.getExpression() == null + || expressionMatcher.match(attributeMap, criteria.getExpression())) { list.add(meta); } } catch (final RuntimeException e) { @@ -248,27 +249,27 @@ public SelectionSummary getReprocessSelectionSummary(final FindMetaCriteria crit /** * Turns a data row object into a generic map of attributes for use by an expression filter. */ - private static Map createAttributeMap(final Meta meta) { - final Map attributeMap = new HashMap<>(); + private static Map createAttributeMap(final Meta meta) { + final Map attributeMap = new HashMap<>(); if (meta != null) { - attributeMap.put(MetaFields.ID.getFldName(), meta.getId()); - attributeMap.put(MetaFields.CREATE_TIME.getFldName(), meta.getCreateMs()); - attributeMap.put(MetaFields.EFFECTIVE_TIME.getFldName(), meta.getEffectiveMs()); - attributeMap.put(MetaFields.STATUS_TIME.getFldName(), meta.getStatusMs()); - attributeMap.put(MetaFields.STATUS.getFldName(), meta.getStatus().getDisplayValue()); + attributeMap.put(CIKey.of(MetaFields.ID.getFldName()), meta.getId()); + attributeMap.put(CIKey.of(MetaFields.CREATE_TIME.getFldName()), meta.getCreateMs()); + attributeMap.put(CIKey.of(MetaFields.EFFECTIVE_TIME.getFldName()), meta.getEffectiveMs()); + attributeMap.put(CIKey.of(MetaFields.STATUS_TIME.getFldName()), meta.getStatusMs()); + attributeMap.put(CIKey.of(MetaFields.STATUS.getFldName()), meta.getStatus().getDisplayValue()); if (meta.getParentMetaId() != null) { - attributeMap.put(MetaFields.PARENT_ID.getFldName(), meta.getParentMetaId()); + attributeMap.put(CIKey.of(MetaFields.PARENT_ID.getFldName()), meta.getParentMetaId()); } if (meta.getTypeName() != null) { - attributeMap.put(MetaFields.TYPE.getFldName(), meta.getTypeName()); + attributeMap.put(CIKey.of(MetaFields.TYPE.getFldName()), meta.getTypeName()); } final String feedName = meta.getFeedName(); if (feedName != null) { - attributeMap.put(MetaFields.FEED.getFldName(), feedName); + attributeMap.put(CIKey.of(MetaFields.FEED.getFldName()), feedName); } final String pipelineUuid = meta.getPipelineUuid(); - attributeMap.put(MetaFields.PIPELINE.getFldName(), pipelineUuid); + attributeMap.put(CIKey.of(MetaFields.PIPELINE.getFldName()), pipelineUuid); // if (processor != null) { // final String pipelineUuid = processor.getPipelineUuid(); // if (pipelineUuid != null) { diff --git a/stroom-meta/stroom-meta-statistics-api/build.gradle b/stroom-meta/stroom-meta-statistics-api/build.gradle index bfd06793c0a..829784054ab 100644 --- a/stroom-meta/stroom-meta-statistics-api/build.gradle +++ b/stroom-meta/stroom-meta-statistics-api/build.gradle @@ -1 +1,5 @@ -ext.moduleName = 'stroom.meta.statistics.api' \ No newline at end of file +ext.moduleName = 'stroom.meta.statistics.api' + +dependencies { + implementation project(':stroom-util-shared') +} diff --git a/stroom-meta/stroom-meta-statistics-api/src/main/java/stroom/meta/statistics/api/MetaStatistics.java b/stroom-meta/stroom-meta-statistics-api/src/main/java/stroom/meta/statistics/api/MetaStatistics.java index dbff0b165d0..2f9f2acad04 100644 --- a/stroom-meta/stroom-meta-statistics-api/src/main/java/stroom/meta/statistics/api/MetaStatistics.java +++ b/stroom-meta/stroom-meta-statistics-api/src/main/java/stroom/meta/statistics/api/MetaStatistics.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,11 @@ package stroom.meta.statistics.api; +import stroom.util.shared.string.CIKey; + import java.util.Map; public interface MetaStatistics { - void recordStatistics(Map metaData); + void recordStatistics(Map metaData); } diff --git a/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsImpl.java b/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsImpl.java index c2ec4c36db6..a81765675de 100644 --- a/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsImpl.java +++ b/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import stroom.statistics.api.InternalStatisticEvent; import stroom.statistics.api.InternalStatisticsReceiver; import stroom.util.date.DateUtil; +import stroom.util.shared.string.CIKey; import com.google.common.base.Preconditions; import jakarta.inject.Inject; @@ -52,7 +53,7 @@ class MetaStatisticsImpl implements MetaStatistics { } @Override - public void recordStatistics(final Map metaData) { + public void recordStatistics(final Map metaData) { securityContext.asProcessingUser(() -> { final InternalStatisticsReceiver receiver = internalStatisticsReceiverProvider.get(); if (receiver != null) { @@ -76,7 +77,7 @@ public void recordStatistics(final Map metaData) { * @return build the STAT or return null for not valid */ private InternalStatisticEvent buildStatisticEvent(final MetaStatisticsTemplate template, - final Map metaData) { + final Map metaData) { Preconditions.checkNotNull(template); Preconditions.checkNotNull(template.getTimeMsAttribute()); Preconditions.checkNotNull(template.getKey()); @@ -98,11 +99,11 @@ private InternalStatisticEvent buildStatisticEvent(final MetaStatisticsTemplate final SortedMap statisticTags = new TreeMap<>(); - for (final String tagName : template.getTagAttributeList()) { + for (final CIKey tagName : template.getTagAttributeList()) { final String tagValue = metaData.get(tagName); if (tagValue != null && !tagValue.isEmpty()) { - statisticTags.put(tagName, tagValue); + statisticTags.put(tagName.get(), tagValue); } else { // Quit! return null; diff --git a/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsModule.java b/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsModule.java index d1ddfebae49..79497f84ba3 100644 --- a/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsModule.java +++ b/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsModule.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Crown Copyright + * Copyright 2018-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,13 +20,14 @@ import stroom.security.api.SecurityContext; import stroom.statistics.api.InternalStatisticKey; import stroom.statistics.api.InternalStatisticsReceiver; +import stroom.util.shared.string.CIKeys; import com.google.inject.AbstractModule; import com.google.inject.Provides; import jakarta.inject.Provider; -import java.util.Arrays; import java.util.Collections; +import java.util.List; public class MetaStatisticsModule extends AbstractModule { @@ -43,16 +44,16 @@ public MetaStatisticsImpl metaStatistics( final MetaStatisticsImpl metaDataStatistic = new MetaStatisticsImpl( internalStatisticsReceiverProvider, securityContext); - metaDataStatistic.setTemplates(Arrays.asList( + metaDataStatistic.setTemplates(List.of( new MetaStatisticsTemplate( InternalStatisticKey.METADATA_STREAMS_RECEIVED, - "receivedTime", - Collections.singletonList("Feed")), + CIKeys.RECEIVED_TIME, + Collections.singletonList(CIKeys.FEED)), new MetaStatisticsTemplate( InternalStatisticKey.METADATA_STREAM_SIZE, - "receivedTime", - "StreamSize", - Collections.singletonList("Feed")))); + CIKeys.RECEIVED_TIME, + CIKeys.STREAM_SIZE, + Collections.singletonList(CIKeys.FEED)))); return metaDataStatistic; } } diff --git a/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsTemplate.java b/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsTemplate.java index 7af79641b86..c1c27de2d9d 100644 --- a/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsTemplate.java +++ b/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MetaStatisticsTemplate.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package stroom.meta.statistics.impl; import stroom.statistics.api.InternalStatisticKey; +import stroom.util.shared.string.CIKey; import java.io.Serializable; import java.util.List; @@ -29,41 +30,42 @@ class MetaStatisticsTemplate implements Serializable { private static final long serialVersionUID = -2347332113575225973L; private InternalStatisticKey key; - private String timeMsAttribute; - private String incrementAttribute; - private List tagAttributeList; + private CIKey timeMsAttribute; + private CIKey incrementAttribute; + private List tagAttributeList; MetaStatisticsTemplate() { } - MetaStatisticsTemplate(final InternalStatisticKey key, final String timeMsAttribute, - final List tagAttributeList) { + MetaStatisticsTemplate(final InternalStatisticKey key, + final CIKey timeMsAttribute, + final List tagAttributeList) { this(key, timeMsAttribute, null, tagAttributeList); } MetaStatisticsTemplate(final InternalStatisticKey key, - final String timeMsAttribute, - final String incrementAttribute, - final List tagAttributeList) { + final CIKey timeMsAttribute, + final CIKey incrementAttribute, + final List tagAttributeList) { this.key = key; this.timeMsAttribute = timeMsAttribute; this.incrementAttribute = incrementAttribute; this.tagAttributeList = tagAttributeList; } - public String getTimeMsAttribute() { + public CIKey getTimeMsAttribute() { return timeMsAttribute; } - public void setTimeMsAttribute(final String timeMsAttribute) { + public void setTimeMsAttribute(final CIKey timeMsAttribute) { this.timeMsAttribute = timeMsAttribute; } - public List getTagAttributeList() { + public List getTagAttributeList() { return tagAttributeList; } - public void setTagAttributeList(final List tagAttributeList) { + public void setTagAttributeList(final List tagAttributeList) { this.tagAttributeList = tagAttributeList; } @@ -75,11 +77,21 @@ public void setKey(final InternalStatisticKey key) { this.key = key; } - public String getIncrementAttribute() { + public CIKey getIncrementAttribute() { return incrementAttribute; } - public void setIncrementAttribute(final String incrementAttribute) { + public void setIncrementAttribute(final CIKey incrementAttribute) { this.incrementAttribute = incrementAttribute; } + + @Override + public String toString() { + return "MetaStatisticsTemplate{" + + "key=" + key + + ", timeMsAttribute=" + timeMsAttribute + + ", incrementAttribute=" + incrementAttribute + + ", tagAttributeList=" + tagAttributeList + + '}'; + } } diff --git a/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MockMetaStatistics.java b/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MockMetaStatistics.java index 284a6eb4af5..6e5fe55d686 100644 --- a/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MockMetaStatistics.java +++ b/stroom-meta/stroom-meta-statistics-impl/src/main/java/stroom/meta/statistics/impl/MockMetaStatistics.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,12 +17,13 @@ package stroom.meta.statistics.impl; import stroom.meta.statistics.api.MetaStatistics; +import stroom.util.shared.string.CIKey; import java.util.Map; class MockMetaStatistics implements MetaStatistics { @Override - public void recordStatistics(Map metaData) { + public void recordStatistics(Map metaData) { } } diff --git a/stroom-node/stroom-node-impl-db/src/main/java/stroom/node/impl/db/NodeDaoImpl.java b/stroom-node/stroom-node-impl-db/src/main/java/stroom/node/impl/db/NodeDaoImpl.java index 2a1a385491f..368e1e66d72 100644 --- a/stroom-node/stroom-node-impl-db/src/main/java/stroom/node/impl/db/NodeDaoImpl.java +++ b/stroom-node/stroom-node-impl-db/src/main/java/stroom/node/impl/db/NodeDaoImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Crown Copyright + * Copyright 2018-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import stroom.node.impl.db.jooq.tables.records.NodeRecord; import stroom.node.shared.Node; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import org.jooq.Condition; @@ -37,7 +38,7 @@ public class NodeDaoImpl implements NodeDao { - private static final Map> FIELD_MAP = Map.of( + private static final Map> FIELD_MAP = CIKey.mapOf( FindNodeCriteria.FIELD_ID, NODE.ID, FindNodeCriteria.FIELD_LAST_BOOT_MS, NODE.LAST_BOOT_MS, FindNodeCriteria.FIELD_BUILD_VERSION, NODE.BUILD_VERSION, diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/PipelineServiceImpl.java b/stroom-pipeline/src/main/java/stroom/pipeline/PipelineServiceImpl.java index 12aa6bf2627..c91512becda 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/PipelineServiceImpl.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/PipelineServiceImpl.java @@ -146,7 +146,7 @@ public List fetchPipelineData(final DocRef pipeline) { @Override public List findUuidsByName(final String nameFilter) { return securityContext.secureResult(() -> - pipelineStore.findByName(nameFilter, true) + pipelineStore.findByName(nameFilter, true, false) .stream() .map(DocRef::getUuid) .collect(Collectors.toList())); diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/PipelineStoreImpl.java b/stroom-pipeline/src/main/java/stroom/pipeline/PipelineStoreImpl.java index 5d7f12db7af..ce4e5edcf3b 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/PipelineStoreImpl.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/PipelineStoreImpl.java @@ -256,8 +256,10 @@ public String getType() { //////////////////////////////////////////////////////////////////////// @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/destination/RollingStreamDestination.java b/stroom-pipeline/src/main/java/stroom/pipeline/destination/RollingStreamDestination.java index 0909c86be0a..c199cfa9c4c 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/destination/RollingStreamDestination.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/destination/RollingStreamDestination.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -81,7 +81,9 @@ protected void beforeRoll(final Consumer exceptionConsumer) { @Override protected void afterRoll(final Consumer exceptionConsumer) { try { - streamTarget.getAttributes().put(MetaFields.REC_WRITE.getFldName(), recordCount.toString()); + streamTarget.getAttributes().put( + MetaFields.REC_WRITE.getFldNameAsCIKey(), + recordCount.toString()); streamTarget.close(); } catch (final IOException e) { throw new RuntimeException(e.getMessage(), e); diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/filter/DocFinder.java b/stroom-pipeline/src/main/java/stroom/pipeline/filter/DocFinder.java index 63e900fbf71..9dbc7894f55 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/filter/DocFinder.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/filter/DocFinder.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.pipeline.filter; import stroom.docref.DocRef; @@ -6,6 +22,7 @@ import stroom.docrefinfo.api.DocRefInfoService; import stroom.docstore.shared.Doc; import stroom.pipeline.errorhandler.ProcessException; +import stroom.util.NullSafe; import stroom.util.io.PathCreator; import stroom.util.logging.LogUtil; @@ -52,7 +69,7 @@ public DocRef findDoc(final DocRef defaultRef, // Load the document from a name pattern if one has been specified. - if (namePattern != null && namePattern.trim().length() > 0) { + if (NullSafe.isNonBlankString(namePattern)) { // Resolve replacement variables. String resolvedName = namePattern.trim(); if (feedName != null) { @@ -83,7 +100,7 @@ public DocRef findDoc(final DocRef defaultRef, } final List docs = hasFindDocsByName.findByName(resolvedName); - if (docs == null || docs.size() == 0) { + if (NullSafe.hasItems(docs)) { if (errorConsumer != null && !suppressNotFoundWarnings) { final StringBuilder sb = new StringBuilder() .append("No ") @@ -109,9 +126,8 @@ public DocRef findDoc(final DocRef defaultRef, } else { doc = docs.get(0); if (errorConsumer != null && docs.size() > 1) { - final String message = "" + - "Found " + docs.size() - + " " + type + "s with name '" + + final String message = "Found " + docs.size() + + " " + type + "s with name '" + resolvedName + "' from pattern '" + namePattern + diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/refdata/ReferenceDataServiceImpl.java b/stroom-pipeline/src/main/java/stroom/pipeline/refdata/ReferenceDataServiceImpl.java index ba2c8d1f1a3..4651f3d02cd 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/refdata/ReferenceDataServiceImpl.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/refdata/ReferenceDataServiceImpl.java @@ -67,6 +67,7 @@ import stroom.util.shared.PermissionException; import stroom.util.shared.ResourcePaths; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import stroom.util.time.StroomDuration; import com.google.common.base.Strings; @@ -99,10 +100,10 @@ public class ReferenceDataServiceImpl implements ReferenceDataService { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(ReferenceDataServiceImpl.class); - private static final Map FIELD_NAME_TO_FIELD_MAP = ReferenceDataFields.FIELDS.stream() - .collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + private static final Map FIELD_NAME_TO_FIELD_MAP = ReferenceDataFields.FIELDS.stream() + .collect(Collectors.toMap(QueryField::getFldNameAsCIKey, Function.identity())); - private static final Map> FIELD_TO_EXTRACTOR_MAP = Map.ofEntries( + private static final Map> FIELD_TO_EXTRACTOR_MAP = CIKey.mapOfEntries( Map.entry(ReferenceDataFields.FEED_NAME_FIELD.getFldName(), RefStoreEntry::getFeedName), Map.entry(ReferenceDataFields.KEY_FIELD.getFldName(), @@ -780,14 +781,15 @@ private void doSearch(final ExpressionCriteria criteria, throw new TaskTerminatedException(); } if (skipTest.getAsBoolean()) { - final String[] fields = fieldIndex.getFields(); - final Val[] valArr = new Val[fields.length]; + final List fields = fieldIndex.getFieldsAsCIKeys(); + final int fieldCount = fields.size(); + final Val[] valArr = new Val[fieldCount]; // Useful for slowing down the search in dev to test termination //ThreadUtil.sleepIgnoringInterrupts(50); - for (int i = 0; i < fields.length; i++) { - final String fieldName = fields[i]; + for (int i = 0; i < fieldCount; i++) { + final CIKey fieldName = fields.get(i); final QueryField field = FIELD_NAME_TO_FIELD_MAP.get(fieldName); // May be a custom field that we obvs can't extract if (field != null) { @@ -964,17 +966,18 @@ private Predicate convertExpressionTerm(final ExpressionTerm expr // name => field // field => fieldType - QueryField abstractField = FIELD_NAME_TO_FIELD_MAP.get(expressionTerm.getField()); + final CIKey fieldName = CIKey.of(expressionTerm.getField()); + QueryField abstractField = FIELD_NAME_TO_FIELD_MAP.get(fieldName); return switch (abstractField.getFldType()) { case TEXT -> buildTextFieldPredicate(expressionTerm, refStoreEntry -> - (String) FIELD_TO_EXTRACTOR_MAP.get(expressionTerm.getField()).apply(refStoreEntry)); + (String) FIELD_TO_EXTRACTOR_MAP.get(fieldName).apply(refStoreEntry)); case LONG -> buildLongFieldPredicate(expressionTerm, refStoreEntry -> - (Long) FIELD_TO_EXTRACTOR_MAP.get(expressionTerm.getField()).apply(refStoreEntry)); + (Long) FIELD_TO_EXTRACTOR_MAP.get(fieldName).apply(refStoreEntry)); case DATE -> buildDateFieldPredicate(expressionTerm, refStoreEntry -> - (Long) FIELD_TO_EXTRACTOR_MAP.get(expressionTerm.getField()).apply(refStoreEntry)); + (Long) FIELD_TO_EXTRACTOR_MAP.get(fieldName).apply(refStoreEntry)); case DOC_REF -> buildDocRefFieldPredicate(expressionTerm, refStoreEntry -> - (DocRef) FIELD_TO_EXTRACTOR_MAP.get(expressionTerm.getField()).apply(refStoreEntry)); + (DocRef) FIELD_TO_EXTRACTOR_MAP.get(fieldName).apply(refStoreEntry)); default -> throw new RuntimeException("Unsupported term " + expressionTerm); }; diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/stepping/SteppingService.java b/stroom-pipeline/src/main/java/stroom/pipeline/stepping/SteppingService.java index 5018cbed1af..1862fff0026 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/stepping/SteppingService.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/stepping/SteppingService.java @@ -223,7 +223,7 @@ private Meta getMeta(final Long id) { return securityContext.asProcessingUserResult(() -> { final FindMetaCriteria criteria = FindMetaCriteria.createFromId(id); final List streamList = metaService.find(criteria).getValues(); - return NullSafe.first(streamList); + return NullSafe.first(streamList).orElse(null); }); } } diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/task/ProcessStatisticsFactory.java b/stroom-pipeline/src/main/java/stroom/pipeline/task/ProcessStatisticsFactory.java index 6bc00c69c46..f9b05fe80a2 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/task/ProcessStatisticsFactory.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/task/ProcessStatisticsFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Crown Copyright + * Copyright 2018-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ import java.util.Map; public class ProcessStatisticsFactory { + public static ProcessStatistics create(final RecordCount recordCount, final ErrorReceiverProxy errorReceiverProxy) { ErrorStatistics errorStatistics = null; @@ -68,11 +69,16 @@ private static long getMarkerCount(final ErrorStatistics errorStatistics, return count; } + + // -------------------------------------------------------------------------------- + + public static class ProcessStatistics { + private final Map map = new HashMap<>(); public void write(final AttributeMap attributeMap) { - map.forEach((k, v) -> attributeMap.put(k.getFldName(), String.valueOf(v))); + map.forEach((k, v) -> attributeMap.put(k.getFldNameAsCIKey(), String.valueOf(v))); } public ProcessStatistics add(final ProcessStatistics stats) { diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/task/StreamMetaDataProvider.java b/stroom-pipeline/src/main/java/stroom/pipeline/task/StreamMetaDataProvider.java index 1dd5d1afc1b..b1d84226469 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/task/StreamMetaDataProvider.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/task/StreamMetaDataProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.pipeline.task; @@ -28,6 +27,7 @@ import stroom.pipeline.shared.PipelineDoc; import stroom.pipeline.state.MetaDataProvider; import stroom.pipeline.state.MetaHolder; +import stroom.util.NullSafe; import stroom.util.date.DateUtil; import java.io.IOException; @@ -59,7 +59,7 @@ public StreamMetaDataProvider(final MetaHolder metaHolder, @Override public String get(final String key) { - if (key == null || key.length() == 0) { + if (NullSafe.isEmptyString(key)) { return null; } diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/textconverter/TextConverterStoreImpl.java b/stroom-pipeline/src/main/java/stroom/pipeline/textconverter/TextConverterStoreImpl.java index e64020ccf65..84fd1967ea0 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/textconverter/TextConverterStoreImpl.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/textconverter/TextConverterStoreImpl.java @@ -184,8 +184,10 @@ public Set findAssociatedNonExplorerDocRefs(DocRef docRef) { //////////////////////////////////////////////////////////////////////// @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/writer/CSVFormatter.java b/stroom-pipeline/src/main/java/stroom/pipeline/writer/CSVFormatter.java index 09806ed2582..27b67c0d3d2 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/writer/CSVFormatter.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/writer/CSVFormatter.java @@ -1,12 +1,33 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.pipeline.writer; +import stroom.meta.api.AttributeMap; +import stroom.util.NullSafe; + import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.regex.Pattern; public class CSVFormatter { + private static final String COMMA = ","; private static final String QUOTE = "\""; private static final String DOUBLE_QUOTE = "\"\""; @@ -38,6 +59,32 @@ public static String format(final Map map) { return sb.toString(); } + public static String format(final AttributeMap map) { + if (NullSafe.hasEntries(map)) { + final StringBuilder sb = new StringBuilder(); + map.entrySet() + .stream() + .sorted(Entry.comparingByKey()) + .forEach(entry -> { + final String key = escape(entry.getKey().get()); + final String value = escape(entry.getValue()); + sb.append(QUOTE); + sb.append(escape(key)); + sb.append(EQUALS); + sb.append(escape(value)); + sb.append(QUOTE); + sb.append(COMMA); + }); + + if (!sb.isEmpty()) { + sb.setLength(sb.length() - 1); + } + return sb.toString(); + } else { + return ""; + } + } + public static String escape(final String value) { if (value == null) { return ""; diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/writer/HTTPAppender.java b/stroom-pipeline/src/main/java/stroom/pipeline/writer/HTTPAppender.java index f4ba2bcc36b..7e9c5113d44 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/writer/HTTPAppender.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/writer/HTTPAppender.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.pipeline.writer; import stroom.meta.api.AttributeMap; @@ -20,6 +36,7 @@ import stroom.util.logging.LogUtil; import stroom.util.shared.ModelStringUtil; import stroom.util.shared.Severity; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import org.apache.commons.compress.compressors.CompressorStreamFactory; @@ -102,7 +119,7 @@ public class HTTPAppender extends AbstractAppender { private Long connectionTimeout; private Long readTimeout; private Long forwardChunkSize; - private Set metaKeySet = getMetaKeySet("guid,feed,system,environment,remotehost,remoteaddress"); + private Set metaKeySet = getMetaKeySet("guid,feed,system,environment,remotehost,remoteaddress"); private HttpURLConnection connection; private final OutputFactory outputStreamSupport; @@ -118,7 +135,7 @@ public class HTTPAppender extends AbstractAppender { private String httpHeadersUserDefinedHeader1; private String httpHeadersUserDefinedHeader2; private String httpHeadersUserDefinedHeader3; - private final Map> requestProperties = new HashMap<>(); + private final Map> requestProperties = new HashMap<>(); // Comma delimited meta keys private String httpHeadersStreamMetaDataAllowList; // Comma delimited meta keys @@ -193,7 +210,7 @@ protected Output createOutput() { setCompressionProperties(outputStreamSupport, connection); - for (Entry entry : effectiveAttributeMap.entrySet()) { + for (Entry entry : effectiveAttributeMap.entrySet()) { addRequestProperty(connection, entry.getKey(), entry.getValue()); } @@ -304,16 +321,16 @@ private AttributeMap cloneFromStreamMeta() { // Allow trumps deny if (NullSafe.isNonBlankString(httpHeadersStreamMetaDataAllowList)) { effectiveAttributeMap = new AttributeMap(); - final Set allowSet = keysToSet(httpHeadersStreamMetaDataAllowList); + final Set allowSet = keysToSet(httpHeadersStreamMetaDataAllowList); clonedAttributeMap.forEach((key, value) -> { - if (allowSet.contains(key.toLowerCase())) { + if (allowSet.contains(key)) { effectiveAttributeMap.put(key, value); } }); } else { effectiveAttributeMap = clonedAttributeMap; if (NullSafe.isNonBlankString(httpHeadersStreamMetaDataDenyList)) { - final Set denySet = keysToSet(httpHeadersStreamMetaDataDenyList); + final Set denySet = keysToSet(httpHeadersStreamMetaDataDenyList); effectiveAttributeMap.removeAll(denySet); } } @@ -324,7 +341,7 @@ private AttributeMap cloneFromStreamMeta() { } private String attributeMapToLines(final AttributeMap attributeMap, final String indent) { - if (NullSafe.isEmptyMap(attributeMap)) { + if (NullSafe.isTrue(attributeMap, AttributeMap::isEmpty)) { return ""; } else { return attributeMap.entrySet() @@ -336,7 +353,7 @@ private String attributeMapToLines(final AttributeMap attributeMap, final String } } - private Set keysToSet(final String keys) { + private Set keysToSet(final String keys) { if (NullSafe.isBlankString(keys)) { return Collections.emptySet(); } else { @@ -344,7 +361,7 @@ private Set keysToSet(final String keys) { .filter(Objects::nonNull) .filter(Predicate.not(String::isBlank)) .map(String::trim) - .map(String::toLowerCase) + .map(CIKey::of) .collect(Collectors.toSet()); } } @@ -425,14 +442,14 @@ public void close() throws IOException { } private void addRequestProperty(final HttpURLConnection connection, - final String key, + final CIKey key, final String value) { - connection.addRequestProperty(key, value); + connection.addRequestProperty(key.get(), value); // It's not possible to inspect the connection to see what req props have been set // as that implicitly opens the connection, so store them in our own map for logging later if (LOGGER.isDebugEnabled()) { - if (key != null) { + if (!CIKey.isNull(key)) { requestProperties.computeIfAbsent(key, k -> new HashSet<>()) .add(value); } else { @@ -515,9 +532,7 @@ private void log(final Logger logger, final long bytes, final long duration) { if (logger.isInfoEnabled() && !metaKeySet.isEmpty()) { - final Map filteredMap = attributeMap.entrySet().stream() - .filter(entry -> metaKeySet.contains(entry.getKey().toLowerCase())) - .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + final AttributeMap filteredMap = attributeMap.filterIncluding(metaKeySet); final String kvPairs = CSVFormatter.format(filteredMap); final String message = CSVFormatter.escape(type) + "," + @@ -534,12 +549,14 @@ private void log(final Logger logger, } } - private Set getMetaKeySet(final String csv) { + private Set getMetaKeySet(final String csv) { if (NullSafe.isEmptyString(csv)) { return Collections.emptySet(); } - return Arrays.stream(csv.toLowerCase().split(",")) + return Arrays.stream(csv.split(",")) + .map(String::trim) + .map(CIKey::ofIgnoringCase) .collect(Collectors.toSet()); } diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/writer/StreamAppender.java b/stroom-pipeline/src/main/java/stroom/pipeline/writer/StreamAppender.java index b790f8e7ddd..c06224ad7ca 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/writer/StreamAppender.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/writer/StreamAppender.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.pipeline.writer; @@ -202,7 +201,9 @@ private void close() { currentStatistics.write(streamTarget.getAttributes()); // Overwrite the actual output record count. - streamTarget.getAttributes().put(MetaFields.REC_WRITE.getFldName(), String.valueOf(count)); + streamTarget.getAttributes().put( + MetaFields.REC_WRITE.getFldNameAsCIKey(), + String.valueOf(count)); // Close the stream target. try { diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/writer/ZipOutput.java b/stroom-pipeline/src/main/java/stroom/pipeline/writer/ZipOutput.java index d48152e9c5f..f3ba5867d4d 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/writer/ZipOutput.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/writer/ZipOutput.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.pipeline.writer; import stroom.meta.api.AttributeMap; @@ -7,6 +23,7 @@ import stroom.util.io.ByteCountOutputStream; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; import com.google.common.base.Strings; import org.apache.commons.compress.archivers.zip.Zip64Mode; @@ -20,6 +37,8 @@ public class ZipOutput implements Output { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(ZipOutput.class); + private static final CIKey FILE_NAME_KEY = CIKey.ofStaticKey("fileName"); + public static final String DATA_EXTENSION = ".dat"; public static final String META_EXTENSION = ".meta"; @@ -90,7 +109,7 @@ public void startZipEntry() throws IOException { if (effectiveAttributeMap != null) { // TODO : I'm not sure where/who is setting fileName in meta so will leave for now. - final String fileName = effectiveAttributeMap.get("fileName"); + final String fileName = effectiveAttributeMap.get(FILE_NAME_KEY); if (!NullSafe.isBlankString(fileName)) { dataFileName = fileName; final int index = fileName.lastIndexOf("."); diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xml/util/XMLWriter.java b/stroom-pipeline/src/main/java/stroom/pipeline/xml/util/XMLWriter.java index 68bd9c3b528..05ba76f1483 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/xml/util/XMLWriter.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xml/util/XMLWriter.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -543,6 +543,10 @@ public String getOutput() { } } + + // -------------------------------------------------------------------------------- + + private static class AttributeNameComparator implements Comparator, Serializable { private static final long serialVersionUID = -9219753718768871842L; diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xmlschema/XmlSchemaStoreImpl.java b/stroom-pipeline/src/main/java/stroom/pipeline/xmlschema/XmlSchemaStoreImpl.java index dced4e02b2d..ebb94cd45e7 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/xmlschema/XmlSchemaStoreImpl.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xmlschema/XmlSchemaStoreImpl.java @@ -214,8 +214,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xslt/XsltStoreImpl.java b/stroom-pipeline/src/main/java/stroom/pipeline/xslt/XsltStoreImpl.java index 1d0bee3d953..74de1666225 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/xslt/XsltStoreImpl.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xslt/XsltStoreImpl.java @@ -184,8 +184,10 @@ public String getType() { //////////////////////////////////////////////////////////////////////// @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/MetaAttribute.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/MetaAttribute.java index a37635b956f..abc4674dc01 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/MetaAttribute.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/MetaAttribute.java @@ -1,8 +1,25 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.pipeline.xsltfunctions; import stroom.data.store.api.DataService; import stroom.pipeline.state.MetaHolder; import stroom.util.shared.Severity; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import net.sf.saxon.expr.XPathContext; @@ -18,7 +35,7 @@ public class MetaAttribute extends StroomExtensionFunctionCall { private final MetaHolder metaHolder; private final DataService dataService; - private Map metaAttributes = null; + private Map metaAttributes = null; @Inject MetaAttribute(final MetaHolder metaHolder, @@ -28,7 +45,9 @@ public class MetaAttribute extends StroomExtensionFunctionCall { } @Override - protected Sequence call(final String functionName, final XPathContext context, final Sequence[] arguments) { + protected Sequence call(final String functionName, + final XPathContext context, + final Sequence[] arguments) { String result = null; try { @@ -39,7 +58,7 @@ protected Sequence call(final String functionName, final XPathContext context, f if (metaAttributes == null) { metaAttributes = dataService.metaAttributes(metaHolder.getMeta().getId()); } - result = metaAttributes.get(key); + result = metaAttributes.get(CIKey.of(key)); } catch (final XPathException | RuntimeException e) { outputWarning(context, new StringBuilder("Error fetching meta attribute for key '" + key + "'"), e); } diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/MetaKeys.java b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/MetaKeys.java index 4eb5b43c0ca..74aff148b24 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/MetaKeys.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/xsltfunctions/MetaKeys.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ protected ArrayItem call(final String functionName, final XPathContext context, try { try { - result = metaDataHolder.getMetaData().keySet(); + result = metaDataHolder.getMetaData().keySetAsStrings(); } catch (final RuntimeException e) { outputWarning(context, new StringBuilder("Error fetching meta keys"), e); } diff --git a/stroom-pipeline/src/test/java/stroom/pipeline/writer/TestCSVFormatter.java b/stroom-pipeline/src/test/java/stroom/pipeline/writer/TestCSVFormatter.java new file mode 100644 index 00000000000..947d5776145 --- /dev/null +++ b/stroom-pipeline/src/test/java/stroom/pipeline/writer/TestCSVFormatter.java @@ -0,0 +1,50 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.pipeline.writer; + +import stroom.meta.api.AttributeMap; +import stroom.util.shared.string.CIKey; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class TestCSVFormatter { + + @Test + void testFormat() { + final Map map = Map.of( + "a", "1", + "b", "2", + "c", "3", + "d", "4"); + + final AttributeMap attributeMap = new AttributeMap(CIKey.mapOf(map)); + + final String str = CSVFormatter.format(map); + final String str2 = CSVFormatter.format(attributeMap); + + assertThat(str) + .isEqualTo(str2); + + assertThat(str) + .isEqualTo(""" + "a=1","b=2","c=3","d=4\""""); + } +} diff --git a/stroom-processor/stroom-processor-impl-db/src/main/java/stroom/processor/impl/db/ProcessorFilterDaoImpl.java b/stroom-processor/stroom-processor-impl-db/src/main/java/stroom/processor/impl/db/ProcessorFilterDaoImpl.java index 00a0ba6d76f..81609c6d000 100644 --- a/stroom-processor/stroom-processor-impl-db/src/main/java/stroom/processor/impl/db/ProcessorFilterDaoImpl.java +++ b/stroom-processor/stroom-processor-impl-db/src/main/java/stroom/processor/impl/db/ProcessorFilterDaoImpl.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.processor.impl.db; import stroom.db.util.ExpressionMapper; @@ -19,6 +35,7 @@ import stroom.util.logging.LogUtil; import stroom.util.shared.ResultPage; import stroom.util.shared.UserRef; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -53,7 +70,7 @@ class ProcessorFilterDaoImpl implements ProcessorFilterDao { private static final Logger LOGGER = LoggerFactory.getLogger(ProcessorFilterDaoImpl.class); private static final LambdaLogger LAMBDA_LOGGER = LambdaLoggerFactory.getLogger(ProcessorFilterDaoImpl.class); - private static final Map> FIELD_MAP = Map.of( + private static final Map> FIELD_MAP = CIKey.mapOf( ProcessorFilterFields.FIELD_ID, PROCESSOR_FILTER.ID); private static final Function RECORD_TO_PROCESSOR_MAPPER = new RecordToProcessorMapper(); diff --git a/stroom-processor/stroom-processor-impl-db/src/main/java/stroom/processor/impl/db/ProcessorTaskDaoImpl.java b/stroom-processor/stroom-processor-impl-db/src/main/java/stroom/processor/impl/db/ProcessorTaskDaoImpl.java index de5b7f610f6..fa365de20c9 100644 --- a/stroom-processor/stroom-processor-impl-db/src/main/java/stroom/processor/impl/db/ProcessorTaskDaoImpl.java +++ b/stroom-processor/stroom-processor-impl-db/src/main/java/stroom/processor/impl/db/ProcessorTaskDaoImpl.java @@ -58,8 +58,10 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; +import stroom.util.shared.GwtNullSafe; import stroom.util.shared.PageRequest; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -80,7 +82,6 @@ import java.sql.SQLIntegrityConstraintViolationException; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -96,7 +97,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static java.util.Map.entry; import static stroom.processor.impl.db.jooq.tables.Processor.PROCESSOR; import static stroom.processor.impl.db.jooq.tables.ProcessorFeed.PROCESSOR_FEED; import static stroom.processor.impl.db.jooq.tables.ProcessorFilter.PROCESSOR_FILTER; @@ -125,19 +125,19 @@ class ProcessorTaskDaoImpl implements ProcessorTaskDao { private static final Field COUNT = DSL.count(); - private static final Map> FIELD_MAP = Map.ofEntries( - entry(ProcessorTaskFields.FIELD_ID, PROCESSOR_TASK.ID), - entry(ProcessorTaskFields.FIELD_CREATE_TIME, PROCESSOR_TASK.CREATE_TIME_MS), - entry(ProcessorTaskFields.FIELD_START_TIME, PROCESSOR_TASK.START_TIME_MS), - entry(ProcessorTaskFields.FIELD_END_TIME_DATE, PROCESSOR_TASK.END_TIME_MS), - entry(ProcessorTaskFields.FIELD_FEED, PROCESSOR_FEED.NAME), - entry(ProcessorTaskFields.FIELD_PRIORITY, PROCESSOR_FILTER.PRIORITY), - entry(ProcessorTaskFields.FIELD_PIPELINE, PROCESSOR.PIPELINE_UUID), - entry(ProcessorTaskFields.FIELD_PIPELINE_NAME, PROCESSOR.PIPELINE_UUID), - entry(ProcessorTaskFields.FIELD_STATUS, PROCESSOR_TASK.STATUS), - entry(ProcessorTaskFields.FIELD_COUNT, COUNT), - entry(ProcessorTaskFields.FIELD_NODE, PROCESSOR_NODE.NAME), - entry(ProcessorTaskFields.FIELD_POLL_AGE, PROCESSOR_FILTER_TRACKER.LAST_POLL_MS) + private static final Map> FIELD_NAME_TO_DB_FIELD_MAP = Map.ofEntries( + CIKey.entry(ProcessorTaskFields.FIELD_ID, PROCESSOR_TASK.ID), + CIKey.entry(ProcessorTaskFields.FIELD_CREATE_TIME, PROCESSOR_TASK.CREATE_TIME_MS), + CIKey.entry(ProcessorTaskFields.FIELD_START_TIME, PROCESSOR_TASK.START_TIME_MS), + CIKey.entry(ProcessorTaskFields.FIELD_END_TIME_DATE, PROCESSOR_TASK.END_TIME_MS), + CIKey.entry(ProcessorTaskFields.FIELD_FEED, PROCESSOR_FEED.NAME), + CIKey.entry(ProcessorTaskFields.FIELD_PRIORITY, PROCESSOR_FILTER.PRIORITY), + CIKey.entry(ProcessorTaskFields.FIELD_PIPELINE, PROCESSOR.PIPELINE_UUID), + CIKey.entry(ProcessorTaskFields.FIELD_PIPELINE_NAME, PROCESSOR.PIPELINE_UUID), + CIKey.entry(ProcessorTaskFields.FIELD_STATUS, PROCESSOR_TASK.STATUS), + CIKey.entry(ProcessorTaskFields.FIELD_COUNT, COUNT), + CIKey.entry(ProcessorTaskFields.FIELD_NODE, PROCESSOR_NODE.NAME), + CIKey.entry(ProcessorTaskFields.FIELD_POLL_AGE, PROCESSOR_FILTER_TRACKER.LAST_POLL_MS) ); private static final Field[] PROCESSOR_TASK_COLUMNS = new Field[]{ @@ -727,7 +727,7 @@ private void log(final CreationState creationState, @Override public ResultPage find(final ExpressionCriteria criteria) { final Condition condition = expressionMapper.apply(criteria.getExpression()); - final Collection> orderFields = JooqUtil.getOrderFields(FIELD_MAP, criteria); + final Collection> orderFields = JooqUtil.getOrderFields(FIELD_NAME_TO_DB_FIELD_MAP, criteria); final int offset = JooqUtil.getOffset(criteria.getPageRequest()); final int limit = JooqUtil.getLimit(criteria.getPageRequest(), true); final Result result = JooqUtil.contextResult(processorDbConnProvider, context -> @@ -789,7 +789,7 @@ private List convert(final Result result) { @Override public ResultPage findSummary(final ExpressionCriteria criteria) { final Condition condition = expressionMapper.apply(criteria.getExpression()); - final Collection> orderFields = JooqUtil.getOrderFields(FIELD_MAP, criteria); + final Collection> orderFields = JooqUtil.getOrderFields(FIELD_NAME_TO_DB_FIELD_MAP, criteria); final PageRequest pageRequest = criteria.getPageRequest(); final int offset = JooqUtil.getOffset(pageRequest); final int limit = JooqUtil.getLimit(pageRequest, true); @@ -834,35 +834,53 @@ public ResultPage findSummary(final ExpressionCriteria cri return ResultPage.createCriterialBasedList(list, criteria); } - private boolean isUsed(final Set fieldSet, - final String[] resultFields, + private boolean isUsed(final CIKey field, + final List resultFields, final ExpressionCriteria criteria) { - return Arrays.stream(resultFields).filter(Objects::nonNull).anyMatch(fieldSet::contains) || - ExpressionUtil.termCount(criteria.getExpression(), fieldSet) > 0; + final boolean isInResultFields = NullSafe.stream(resultFields) + .filter(Objects::nonNull) + .anyMatch(resultField -> Objects.equals(resultField, field)); + + return isInResultFields + || ExpressionUtil.termCount(criteria.getExpression(), field.get()) > 0; + } + + private boolean isUsed(final Set fieldSet, + final List resultFields, + final ExpressionCriteria criteria) { + final boolean isInResultFields = NullSafe.stream(resultFields) + .filter(Objects::nonNull) + .anyMatch(fieldSet::contains); + + return isInResultFields || ExpressionUtil.termCount( + criteria.getExpression(), + fieldSet.stream() + .map(CIKey::get).collect(Collectors.toSet())) > 0; } @Override public void search(final ExpressionCriteria criteria, final FieldIndex fieldIndex, final ValuesConsumer consumer) { - final Set processorFields = Set.of( - ProcessorTaskFields.PROCESSOR_FILTER_ID.getFldName(), - ProcessorTaskFields.PROCESSOR_FILTER_PRIORITY.getFldName()); + final Set processorFields = Set.of( + ProcessorTaskFields.PROCESSOR_FILTER_ID.getFldNameAsCIKey(), + ProcessorTaskFields.PROCESSOR_FILTER_PRIORITY.getFldNameAsCIKey()); validateExpressionTerms(criteria.getExpression()); - final String[] fieldNames = fieldIndex.getFields(); - final boolean nodeUsed = isUsed(Set.of(ProcessorTaskFields.NODE_NAME.getFldName()), fieldNames, criteria); - final boolean feedUsed = isUsed(Set.of(ProcessorTaskFields.FEED.getFldName()), fieldNames, criteria); + final List fieldNames = fieldIndex.getFieldsAsCIKeys(); + final int fieldCount = GwtNullSafe.size(fieldNames); + final boolean nodeUsed = isUsed(ProcessorTaskFields.NODE_NAME.getFldNameAsCIKey(), fieldNames, criteria); + final boolean feedUsed = isUsed(ProcessorTaskFields.FEED.getFldNameAsCIKey(), fieldNames, criteria); final boolean processorFilterUsed = isUsed(processorFields, fieldNames, criteria); final boolean processorUsed = - isUsed(Set.of(ProcessorTaskFields.PROCESSOR_ID.getFldName()), fieldNames, criteria); + isUsed(CIKey.of(ProcessorTaskFields.PROCESSOR_ID.getFldName()), fieldNames, criteria); final boolean pipelineUsed = - isUsed(Set.of(ProcessorTaskFields.PIPELINE.getFldName()), fieldNames, criteria); + isUsed(CIKey.of(ProcessorTaskFields.PIPELINE.getFldName()), fieldNames, criteria); final PageRequest pageRequest = criteria.getPageRequest(); final Condition condition = expressionMapper.apply(criteria.getExpression()); - final Collection> orderFields = JooqUtil.getOrderFields(FIELD_MAP, criteria); + final Collection> orderFields = JooqUtil.getOrderFields(FIELD_NAME_TO_DB_FIELD_MAP, criteria); final List> dbFields = new ArrayList<>(valueMapper.getDbFieldsByName(fieldNames)); final Mapper[] mappers = valueMapper.getMappersForFieldNames(fieldNames); @@ -903,8 +921,8 @@ public void search(final ExpressionCriteria criteria, final Result result = cursor.fetchNext(BATCH_SIZE); result.forEach(r -> { - final Val[] arr = new Val[fieldNames.length]; - for (int i = 0; i < fieldNames.length; i++) { + final Val[] arr = new Val[fieldCount]; + for (int i = 0; i < fieldCount; i++) { Val val = ValNull.INSTANCE; final Mapper mapper = mappers[i]; if (mapper != null) { @@ -938,7 +956,7 @@ public ResultPage changeTaskStatus(final ExpressionCriteria crite } final Condition condition = expressionMapper.apply(criteria.getExpression()); - final Collection> orderFields = JooqUtil.getOrderFields(FIELD_MAP, criteria); + final Collection> orderFields = JooqUtil.getOrderFields(FIELD_NAME_TO_DB_FIELD_MAP, criteria); final int offset = JooqUtil.getOffset(criteria.getPageRequest()); final int limit = JooqUtil.getLimit(criteria.getPageRequest(), true); @@ -1120,21 +1138,23 @@ private boolean validateExpressionTerms(final ExpressionItem expressionItem) { if (expressionItem == null) { return true; } else { - final Map fieldMap = ProcessorTaskFields.getFieldMap(); + final Map fieldMap = ProcessorTaskFields.getFieldMap(); return ExpressionUtil.validateExpressionTerms(expressionItem, term -> { - final QueryField field = fieldMap.get(term.getField()); + final CIKey fieldName = CIKey.of(term.getField()); + final QueryField field = fieldMap.get(fieldName); if (field == null) { throw new RuntimeException(LogUtil.message("Unknown field {} in term {}, in expression {}", - term.getField(), term, expressionItem)); + fieldName, term, expressionItem)); } else { final boolean isValid = field.supportsCondition(term.getCondition()); if (!isValid) { throw new RuntimeException(LogUtil.message("Condition '{}' is not supported by field '{}' " + "of type {}. Term: {}", term.getCondition(), - term.getField(), - field.getFldType(), term)); + fieldName, + field.getFldType(), + term)); } else { return true; } @@ -1146,7 +1166,11 @@ private boolean validateExpressionTerms(final ExpressionItem expressionItem) { private List getPipelineUuidsByName(final List pipelineNames) { // Can't cache this in a simple map due to pipes being renamed, but // docRefInfoService should cache most of this anyway. - return docRefInfoService.findByNames(PipelineDoc.DOCUMENT_TYPE, pipelineNames, true) + return docRefInfoService.findByNames( + PipelineDoc.DOCUMENT_TYPE, + pipelineNames, + true, + false) .stream() .map(DocRef::getUuid) .collect(Collectors.toList()); diff --git a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/SqsConnector.java b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/SqsConnector.java index b2bc34d89bb..54c4e42eb5e 100644 --- a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/SqsConnector.java +++ b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/SqsConnector.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.app; import stroom.meta.api.AttributeMap; @@ -76,7 +92,7 @@ public void poll() { if (message.hasAttributes()) { final Map attributesAsStrings = message.attributesAsStrings(); LOGGER.debug(() -> "Attributes: " + attributesAsStrings); - attributeMap.putAll(attributesAsStrings); + attributeMap.putAllWithStringKeys(attributesAsStrings); } LOGGER.debug(() -> "Has Message Attributes: " + message.hasMessageAttributes()); diff --git a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/EventSerialiser.java b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/EventSerialiser.java index a0694d75860..83a7b396cd8 100644 --- a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/EventSerialiser.java +++ b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/EventSerialiser.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.app.event; import stroom.meta.api.AttributeMap; @@ -21,7 +37,7 @@ public String serialise(final String requestUuid, .entrySet() .stream() .sorted(Map.Entry.comparingByKey()) - .map(entry -> new Header(entry.getKey(), entry.getValue())) + .map(entry -> new Header(entry.getKey().get(), entry.getValue())) .toList(); final Event event = new Event( diff --git a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/EventStore.java b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/EventStore.java index 17393a93d44..1c80c767eda 100644 --- a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/EventStore.java +++ b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/EventStore.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.app.event; import stroom.meta.api.AttributeMap; @@ -13,6 +29,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.Metrics; +import stroom.util.shared.string.CIKeys; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -213,8 +230,8 @@ public void consume(final AttributeMap attributeMap, final String requestUuid, final String data) { try { - final String feed = attributeMap.get("Feed"); - final String type = attributeMap.get("type"); + final String feed = attributeMap.get(CIKeys.FEED); + final String type = attributeMap.get(CIKeys.TYPE); final FeedKey feedKey = new FeedKey(feed, type); final String string = eventSerialiser.serialise( diff --git a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/KafkaEventConsumer.java b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/KafkaEventConsumer.java index df544c007e3..3f580fb6d22 100644 --- a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/KafkaEventConsumer.java +++ b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/event/KafkaEventConsumer.java @@ -1,9 +1,26 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.app.event; import stroom.meta.api.AttributeMap; import stroom.proxy.app.ProxyConfig; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKeys; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; @@ -79,12 +96,12 @@ public void consume(final AttributeMap attributeMap, try { final List
headers = new ArrayList<>(attributeMap.size()); attributeMap.forEach((k, v) -> { - final Header header = new RecordHeader(k, v.getBytes(StandardCharsets.UTF_8)); + final Header header = new RecordHeader(k.get(), v.getBytes(StandardCharsets.UTF_8)); headers.add(header); }); - final String feed = attributeMap.get("Feed"); - final String type = attributeMap.get("type"); + final String feed = attributeMap.get(CIKeys.FEED); + final String type = attributeMap.get(CIKeys.TYPE); final FeedKey feedKey = new FeedKey(feed, type); final String string = eventSerialiser.serialise( diff --git a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/forwarder/ForwardHttpPostHandlers.java b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/forwarder/ForwardHttpPostHandlers.java index c1d26c0409c..ea18067b746 100644 --- a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/forwarder/ForwardHttpPostHandlers.java +++ b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/forwarder/ForwardHttpPostHandlers.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.app.forwarder; import stroom.meta.api.AttributeMap; @@ -47,7 +63,7 @@ public void handle(final String feedName, // We need to add the authentication token to our headers final Map authHeaders = userIdentityFactory.getServiceUserAuthHeaders(); - attributeMap.putAll(authHeaders); + attributeMap.putAllWithStringKeys(authHeaders); ForwardStreamHandler streamHandler = null; try { diff --git a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/forwarder/ForwardStreamHandler.java b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/forwarder/ForwardStreamHandler.java index b3e822a75df..d8be7b747e9 100644 --- a/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/forwarder/ForwardStreamHandler.java +++ b/stroom-proxy/stroom-proxy-app/src/main/java/stroom/proxy/app/forwarder/ForwardStreamHandler.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.app.forwarder; import stroom.meta.api.AttributeMap; @@ -15,6 +31,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; import stroom.util.time.StroomDuration; import stroom.util.zip.ZipUtil; @@ -105,11 +122,13 @@ public ForwardStreamHandler(final LogStream logStream, connection.setDoOutput(true); connection.setDoInput(true); - connection.addRequestProperty(StandardHeaderArguments.COMPRESSION, StandardHeaderArguments.COMPRESSION_ZIP); + connection.addRequestProperty( + StandardHeaderArguments.COMPRESSION.get(), + StandardHeaderArguments.COMPRESSION_ZIP); final AttributeMap sendHeader = AttributeMapUtil.cloneAllowable(attributeMap); - for (Entry entry : sendHeader.entrySet()) { - connection.addRequestProperty(entry.getKey(), entry.getValue()); + for (Entry entry : sendHeader.entrySet()) { + connection.addRequestProperty(entry.getKey().get(), entry.getValue()); } // Allows sending to systems on the same OpenId realm as us using an access token @@ -203,14 +222,10 @@ private String formatAttributeMapLogging(final AttributeMap attributeMap) { return attributeMap .entrySet() .stream() - .map(entry -> new SimpleEntry<>( - Objects.requireNonNullElse(entry.getKey(), "null"), - entry.getValue()) - ) .sorted(Entry.comparingByKey()) .map(entry -> " " + String.join( ":", - NullSafe.string(entry.getKey()), + NullSafe.getOrElse(entry.getKey(), CIKey::get, "null"), LogUtil.truncateUnless(entry.getValue(), 50, LOGGER.isTraceEnabled()))) .collect(Collectors.joining("\n")); } diff --git a/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/MockHttpDestination.java b/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/MockHttpDestination.java index 09277430ce8..b320ce707f9 100644 --- a/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/MockHttpDestination.java +++ b/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/MockHttpDestination.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.app; import stroom.meta.api.AttributeMap; @@ -328,7 +344,7 @@ void assertPosts() { // Check feed names. Assertions.assertThat(postsToStroomDataFeed) - .extracting(req -> req.getHeader(StandardHeaderArguments.FEED)) + .extracting(req -> req.getHeader(StandardHeaderArguments.FEED.get())) .containsExactlyInAnyOrder( TestConstants.FEED_TEST_EVENTS_1, TestConstants.FEED_TEST_EVENTS_2, diff --git a/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/TestEndToEndForwardToHttp.java b/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/TestEndToEndForwardToHttp.java index 6a02a65ee32..aed67969c90 100644 --- a/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/TestEndToEndForwardToHttp.java +++ b/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/TestEndToEndForwardToHttp.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.app; import stroom.meta.api.StandardHeaderArguments; @@ -75,7 +91,7 @@ void testBasicEndToEnd() { }); Assertions.assertThat(postsToStroomDataFeed) - .extracting(req -> req.getHeader(StandardHeaderArguments.FEED)) + .extracting(req -> req.getHeader(StandardHeaderArguments.FEED.get())) .containsExactly(TestConstants.FEED_TEST_EVENTS_1, TestConstants.FEED_TEST_EVENTS_2); mockHttpDestination.assertSimpleDataFeedRequestContent(expectedRequestCount); diff --git a/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/TestEventResource.java b/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/TestEventResource.java index 99748349ab9..5d5bde7ffe9 100644 --- a/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/TestEventResource.java +++ b/stroom-proxy/stroom-proxy-app/src/test/java/stroom/proxy/app/TestEventResource.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.app; import stroom.meta.api.StandardHeaderArguments; @@ -89,10 +105,10 @@ private static boolean post(final HttpClient httpClient, try { final HttpPost httpPost = new HttpPost("http://127.0.0.1:8090/api/event"); if (feed != null) { - httpPost.addHeader(StandardHeaderArguments.FEED, feed); + httpPost.addHeader(StandardHeaderArguments.FEED.get(), feed); } if (type != null) { - httpPost.addHeader(StandardHeaderArguments.TYPE, type); + httpPost.addHeader(StandardHeaderArguments.TYPE.get(), type); } httpPost.addHeader("System", "EXAMPLE_SYSTEM"); httpPost.addHeader("Environment", "EXAMPLE_ENVIRONMENT"); diff --git a/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/AggregateForwarder.java b/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/AggregateForwarder.java index d0287250d13..c85c1ec645c 100644 --- a/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/AggregateForwarder.java +++ b/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/AggregateForwarder.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.net.HostNameUtil; +import stroom.util.shared.string.CIKeys; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -43,7 +44,6 @@ public class AggregateForwarder { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(AggregateForwarder.class); - private static final String PROXY_FORWARD_ID = "ProxyForwardId"; private final FeedDao feedDao; private final SourceItemDao sourceItemDao; @@ -172,7 +172,7 @@ public void forward(final ForwardAggregate forwardAggregate) { final AtomicReference error = new AtomicReference<>(); final List items = sourceItemDao.fetchSourceItemsByAggregateId(aggregate.id()); - if (items.size() > 0) { + if (!items.isEmpty()) { final FeedKey feedKey = feedDao.getKey(aggregate.feedId()); final long thisPostId = proxyForwardId.incrementAndGet(); final String info = thisPostId + " " + feedKey.feed() + " - " + feedKey.type(); @@ -186,13 +186,13 @@ public void forward(final ForwardAggregate forwardAggregate) { attributeMap.put(StandardHeaderArguments.TYPE, feedKey.type()); } if (LOGGER.isDebugEnabled()) { - attributeMap.put(PROXY_FORWARD_ID, String.valueOf(thisPostId)); + attributeMap.put(CIKeys.PROXY_FORWARD_ID, String.valueOf(thisPostId)); } final StreamHandlers streamHandlers; // If we have reached the max tried limit then send the data to the failure destination for this forwarder. if (forwardAggregate.getTries() >= forwardRetryConfigProvider.get().getMaxTries()) { - attributeMap.put("ForwardError", forwardAggregate.getError()); + attributeMap.put(CIKeys.FORWARD_ERROR, forwardAggregate.getError()); streamHandlers = failureDestinations.getProvider(forwardAggregate.getForwardDest().getName()); } else { streamHandlers = forwarderDestinations.getProvider(forwardAggregate.getForwardDest().getName()); diff --git a/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/CSVFormatter.java b/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/CSVFormatter.java index 5e77afc6ebe..da56bc8e527 100644 --- a/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/CSVFormatter.java +++ b/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/CSVFormatter.java @@ -1,9 +1,29 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.repo; +import stroom.meta.api.AttributeMap; +import stroom.util.NullSafe; + import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.regex.Pattern; public class CSVFormatter { @@ -32,13 +52,39 @@ public static String format(final Map map) { sb.append(COMMA); } - if (sb.length() > 0) { + if (!sb.isEmpty()) { sb.setLength(sb.length() - 1); } return sb.toString(); } + public static String format(final AttributeMap map) { + if (NullSafe.hasEntries(map)) { + final StringBuilder sb = new StringBuilder(); + map.entrySet() + .stream() + .sorted(Entry.comparingByKey()) + .forEach(entry -> { + final String key = escape(entry.getKey().get()); + final String value = escape(entry.getValue()); + sb.append(QUOTE); + sb.append(escape(key)); + sb.append(EQUALS); + sb.append(escape(value)); + sb.append(QUOTE); + sb.append(COMMA); + }); + + if (!sb.isEmpty()) { + sb.setLength(sb.length() - 1); + } + return sb.toString(); + } else { + return ""; + } + } + public static String escape(final String value) { if (value == null) { return ""; diff --git a/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/LogStream.java b/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/LogStream.java index c339b1b8e40..3bde8cc8ec5 100644 --- a/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/LogStream.java +++ b/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/LogStream.java @@ -1,15 +1,30 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.proxy.repo; import stroom.meta.api.AttributeMap; import stroom.util.NullSafe; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; import jakarta.inject.Singleton; import org.slf4j.Logger; -import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; @@ -43,13 +58,13 @@ public void log(final Logger logger, final String message) { if (logger.isInfoEnabled()) { - final Set metaKeys = logStreamConfigProvider.get().getMetaKeys(); + final Set metaKeys = NullSafe.set(logStreamConfigProvider.get().getMetaKeys()) + .stream() + .map(CIKey::ofIgnoringCase) + .collect(Collectors.toSet()); if (NullSafe.hasItems(metaKeys)) { - final Map filteredMap = attributeMap.entrySet() - .stream() - .filter(entry -> metaKeys.contains(entry.getKey().toLowerCase())) - .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + final AttributeMap filteredMap = attributeMap.filterIncluding(metaKeys); final String kvPairs = CSVFormatter.format(filteredMap); final String logLine = CSVFormatter.escape(type) + "," + diff --git a/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/SourceForwarder.java b/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/SourceForwarder.java index 123848994b7..3e6456b12fd 100644 --- a/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/SourceForwarder.java +++ b/stroom-proxy/stroom-proxy-repo/src/main/java/stroom/proxy/repo/SourceForwarder.java @@ -1,5 +1,5 @@ /* - * Copyright 2019 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.net.HostNameUtil; +import stroom.util.shared.string.CIKeys; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -40,7 +41,6 @@ public class SourceForwarder { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(SourceForwarder.class); - private static final String PROXY_FORWARD_ID = "ProxyForwardId"; private final FeedDao feedDao; private final RepoSources sources; @@ -189,13 +189,13 @@ public void forward(final ForwardSource forwardSource) { attributeMap.put(StandardHeaderArguments.TYPE, feedKey.type()); } if (LOGGER.isDebugEnabled()) { - attributeMap.put(PROXY_FORWARD_ID, String.valueOf(thisPostId)); + attributeMap.put(CIKeys.PROXY_FORWARD_ID, String.valueOf(thisPostId)); } final StreamHandlers streamHandlers; // If we have reached the max tried limit then send the data to the failure destination for this forwarder. if (forwardSource.getTries() >= forwardRetryConfig.getMaxTries()) { - attributeMap.put("ForwardError", forwardSource.getError()); + attributeMap.put(CIKeys.FORWARD_ERROR, forwardSource.getError()); streamHandlers = failureDestinations.getProvider(forwardSource.getForwardDest().getName()); } else { streamHandlers = forwarderDestinations.getProvider(forwardSource.getForwardDest().getName()); diff --git a/stroom-proxy/stroom-proxy-repo/src/test/java/stroom/proxy/repo/TestCSVFormatter.java b/stroom-proxy/stroom-proxy-repo/src/test/java/stroom/proxy/repo/TestCSVFormatter.java new file mode 100644 index 00000000000..c53a3b042ba --- /dev/null +++ b/stroom-proxy/stroom-proxy-repo/src/test/java/stroom/proxy/repo/TestCSVFormatter.java @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.proxy.repo; + +import stroom.meta.api.AttributeMap; +import stroom.pipeline.writer.CSVFormatter; +import stroom.util.shared.string.CIKey; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class TestCSVFormatter { + + @Test + void testFormat() { + final Map map = Map.of( + "a", "1", + "b", "2", + "c", "3", + "d", "4"); + + final AttributeMap attributeMap = new AttributeMap(CIKey.mapOf(map)); + + final String str = stroom.pipeline.writer.CSVFormatter.format(map); + final String str2 = CSVFormatter.format(attributeMap); + + assertThat(str) + .isEqualTo(str2); + + assertThat(str) + .isEqualTo(""" + "a=1","b=2","c=3","d=4\""""); + } +} diff --git a/stroom-query/stroom-query-api/src/main/java/stroom/datasource/api/v2/QueryField.java b/stroom-query/stroom-query-api/src/main/java/stroom/datasource/api/v2/QueryField.java index 18b4044b2ad..8041518266c 100644 --- a/stroom-query/stroom-query-api/src/main/java/stroom/datasource/api/v2/QueryField.java +++ b/stroom-query/stroom-query-api/src/main/java/stroom/datasource/api/v2/QueryField.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,6 +18,8 @@ import stroom.docref.HasDisplayValue; import stroom.query.api.v2.ExpressionTerm.Condition; +import stroom.util.shared.GwtNullSafe; +import stroom.util.shared.string.CIKey; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -59,6 +61,9 @@ public class QueryField implements Field, HasDisplayValue { @JsonProperty private final Boolean queryable; + @JsonIgnore + private transient volatile CIKey fldNameCIKey = null; + @JsonCreator public QueryField(@Deprecated @JsonProperty("type") final String type, @Deprecated @JsonProperty("name") final String name, @@ -75,6 +80,25 @@ public QueryField(@Deprecated @JsonProperty("type") final String type, this.conditionSet = conditionSet; this.docRefType = docRefType; this.queryable = queryable; + this.fldNameCIKey = null; + } + + private QueryField(final String type, + final String name, + final String fldName, + final CIKey fldNameAsKey, + final FieldType fldType, + final ConditionSet conditionSet, + final String docRefType, + final Boolean queryable) { + this.fldName = fldName != null + ? fldName + : name; + this.fldNameCIKey = fldNameAsKey; + this.fldType = convertLegacyType(fldType, type); + this.conditionSet = conditionSet; + this.docRefType = docRefType; + this.queryable = queryable; } private FieldType convertLegacyType(final FieldType fieldType, final String type) { @@ -124,6 +148,16 @@ public static QueryField createId(final String name, .build(); } + public static QueryField createId(final CIKey name, + final Boolean queryable) { + return builder() + .fldName(name) + .fldType(FieldType.ID) + .conditionSet(ConditionSet.DEFAULT_ID) + .queryable(queryable) + .build(); + } + public static QueryField createKeyword(final String name) { return createKeyword(name, true); } @@ -152,6 +186,16 @@ public static QueryField createInteger(final String name, .build(); } + public static QueryField createInteger(final CIKey name, + final Boolean queryable) { + return builder() + .fldName(name) + .fldType(FieldType.INTEGER) + .conditionSet(ConditionSet.DEFAULT_NUMERIC) + .queryable(queryable) + .build(); + } + public static QueryField createLong(final String name) { return createLong(name, true); } @@ -166,6 +210,16 @@ public static QueryField createLong(final String name, .build(); } + public static QueryField createLong(final CIKey name, + final Boolean queryable) { + return builder() + .fldName(name) + .fldType(FieldType.LONG) + .conditionSet(ConditionSet.DEFAULT_NUMERIC) + .queryable(queryable) + .build(); + } + public static QueryField createFloat(final String name) { return createFloat(name, true); } @@ -222,6 +276,16 @@ public static QueryField createBoolean(final String name, .build(); } + public static QueryField createBoolean(final CIKey name, + final Boolean queryable) { + return builder() + .fldName(name) + .fldType(FieldType.BOOLEAN) + .conditionSet(ConditionSet.DEFAULT_BOOLEAN) + .queryable(queryable) + .build(); + } + public static QueryField createDate(final String name) { return createDate(name, true); } @@ -236,6 +300,16 @@ public static QueryField createDate(final String name, .build(); } + public static QueryField createDate(final CIKey name, + final Boolean queryable) { + return builder() + .fldName(name) + .fldType(FieldType.DATE) + .conditionSet(ConditionSet.DEFAULT_DATE) + .queryable(queryable) + .build(); + } + public static QueryField createText(final String name) { return createText(name, true); } @@ -250,6 +324,16 @@ public static QueryField createText(final String name, .build(); } + public static QueryField createText(final CIKey name, + final Boolean queryable) { + return builder() + .fldName(name) + .fldType(FieldType.TEXT) + .conditionSet(ConditionSet.DEFAULT_TEXT) + .queryable(queryable) + .build(); + } + /** * A {@link QueryField} for a {@link stroom.docref.DocRef} type whose names are unique, allowing @@ -266,6 +350,21 @@ public static QueryField createDocRefByUniqueName(final String docRefType, .build(); } + /** + * A {@link QueryField} for a {@link stroom.docref.DocRef} type whose names are unique, allowing + * the name to be used as the value in expression terms. + */ + public static QueryField createDocRefByUniqueName(final String docRefType, + final CIKey name) { + return builder() + .fldName(name) + .fldType(FieldType.DOC_REF) + .conditionSet(ConditionSet.DOC_REF_ALL) + .docRefType(docRefType) + .queryable(Boolean.TRUE) + .build(); + } + /** * A {@link QueryField} for a {@link stroom.docref.DocRef} type whose names are NOT unique. * The {@link stroom.docref.DocRef} name is used as the value in expression terms, accepting @@ -282,6 +381,22 @@ public static QueryField createDocRefByNonUniqueName(final String docRefType, .build(); } + /** + * A {@link QueryField} for a {@link stroom.docref.DocRef} type whose names are NOT unique. + * The {@link stroom.docref.DocRef} name is used as the value in expression terms, accepting + * that name=x may match >1 docrefs. + */ + public static QueryField createDocRefByNonUniqueName(final String docRefType, + final CIKey name) { + return builder() + .fldName(name) + .fldType(FieldType.DOC_REF) + .conditionSet(ConditionSet.DOC_REF_NAME) + .docRefType(docRefType) + .queryable(Boolean.TRUE) + .build(); + } + /** * A {@link QueryField} for a {@link stroom.docref.DocRef} type whose names are NOT unique. * The {@link stroom.docref.DocRef} uuid is used as the value in expression terms for a unique @@ -299,11 +414,38 @@ public static QueryField createDocRefByUuid(final String docRefType, .build(); } + /** + * A {@link QueryField} for a {@link stroom.docref.DocRef} type whose names are NOT unique. + * The {@link stroom.docref.DocRef} uuid is used as the value in expression terms for a unique + * match. Other conditions are not supported as that would require the user to enter uuids, + * and it is not clear in the UI whether they are dealing in UUIDs or names. + */ + public static QueryField createDocRefByUuid(final String docRefType, + final CIKey name) { + return builder() + .fldName(name) + .fldType(FieldType.DOC_REF) + .conditionSet(ConditionSet.DOC_REF_UUID) + .docRefType(docRefType) + .queryable(Boolean.TRUE) + .build(); + } + @Override public String getFldName() { return fldName; } + @JsonIgnore + public CIKey getFldNameAsCIKey() { + if (fldNameCIKey == null) { + final CIKey ciKey = CIKey.of(fldName); + fldNameCIKey = ciKey; + return ciKey; + } + return fldNameCIKey; + } + @Override public FieldType getFldType() { return fldType; @@ -380,9 +522,14 @@ public static Builder builder() { return new Builder(); } + + // -------------------------------------------------------------------------------- + + public static class Builder { private String fldName; + private CIKey fldNameAsKey; private FieldType fldType; private ConditionSet conditionSet; private String docRefType; @@ -393,6 +540,7 @@ private Builder() { private Builder(final QueryField queryField) { this.fldName = queryField.fldName; + this.fldNameAsKey = queryField.getFldNameAsCIKey(); this.fldType = queryField.fldType; this.conditionSet = queryField.conditionSet; this.docRefType = queryField.docRefType; @@ -404,6 +552,12 @@ public Builder fldName(final String fldName) { return this; } + public Builder fldName(final CIKey fldName) { + this.fldName = GwtNullSafe.get(fldName, CIKey::get); + this.fldNameAsKey = fldName; + return this; + } + public Builder fldType(final FieldType fldType) { this.fldType = fldType; return this; @@ -428,10 +582,22 @@ public QueryField build() { if (conditionSet == null && fldType != null) { conditionSet = ConditionSet.getDefault(fldType); } + if (GwtNullSafe.allNonNull(fldName, fldNameAsKey)) { + if (!Objects.equals(fldNameAsKey.get(), fldName)) { + throw new IllegalArgumentException("fldName (" + fldName + + ") and fldNameAsKey (" + + fldNameAsKey.get() + ") do not match"); + } + } else if (fldName != null && fldNameAsKey == null) { + fldNameAsKey = CIKey.of(fldName); + } else if (fldName == null && fldNameAsKey != null) { + fldName = fldNameAsKey.get(); + } return new QueryField( null, null, fldName, + fldNameAsKey, fldType, conditionSet, docRefType, diff --git a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Column.java b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Column.java index a126ac10da2..f1267b4e5b1 100644 --- a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Column.java +++ b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Column.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,6 +17,8 @@ package stroom.query.api.v2; import stroom.docref.HasDisplayValue; +import stroom.util.shared.GwtNullSafe; +import stroom.util.shared.string.CIKey; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -77,6 +79,11 @@ public final class Column implements HasDisplayValue { @JsonProperty private final Boolean special; + @JsonIgnore + private transient volatile CIKey caseInsensitiveId = null; + @JsonIgnore + private transient volatile CIKey caseInsensitiveName; + @JsonCreator public Column(@JsonProperty("id") final String id, @JsonProperty("name") final String name, @@ -95,31 +102,67 @@ public Column(@JsonProperty("id") final String id, this.filter = filter; this.format = format; this.group = group; - if (width != null) { - this.width = width; - } else { - this.width = 200; - } - if (visible != null) { - this.visible = visible; - } else { - this.visible = true; - } - if (special != null) { - this.special = special; - } else { - this.special = false; - } + this.width = GwtNullSafe.requireNonNullElse(width, 200); + this.visible = GwtNullSafe.requireNonNullElse(visible, true); + this.special = GwtNullSafe.requireNonNullElse(special, false); + } + + public Column(final CIKey id, + final CIKey name, + final String expression, + final Sort sort, + final Filter filter, + final Format format, + final Integer group, + final Integer width, + final Boolean visible, + final Boolean special) { + this.caseInsensitiveId = id; + this.id = id.get(); + this.caseInsensitiveName = name; + this.name = name.get(); + this.expression = expression; + this.sort = sort; + this.filter = filter; + this.format = format; + this.group = group; + this.width = GwtNullSafe.requireNonNullElse(width, 200); + this.visible = GwtNullSafe.requireNonNullElse(visible, true); + this.special = GwtNullSafe.requireNonNullElse(special, false); } public String getId() { return id; } + @JsonIgnore + public CIKey getIdAsCIKey() { + if (caseInsensitiveId == null) { + // Saves us building a CIKey every time we need to look up the columns name + final CIKey idKey = CIKey.of(id); + this.caseInsensitiveId = idKey; + return idKey; + } + return caseInsensitiveId; + } + public String getName() { return name; } + @JsonIgnore + public CIKey getNameAsCIKey() { + if (caseInsensitiveName == null) { + // Saves us building a CIKey every time we need to look up the columns name + final CIKey nameKey = Objects.equals(id, name) && this.caseInsensitiveId != null + ? this.caseInsensitiveId + : CIKey.of(name); + this.caseInsensitiveName = nameKey; + return nameKey; + } + return caseInsensitiveName; + } + public String getExpression() { return expression; } @@ -218,13 +261,17 @@ public Builder copy() { return new Builder(this); } + + // -------------------------------------------------------------------------------- + + /** * Builder for constructing a {@link Column} */ public static final class Builder { - private String id; - private String name; + private CIKey id; + private CIKey name; private String expression; private Sort sort; private Filter filter; @@ -251,8 +298,8 @@ private Builder() { } private Builder(final Column field) { - this.id = field.id; - this.name = field.name; + this.id = field.getIdAsCIKey(); + this.name = field.getNameAsCIKey(); this.expression = field.expression; this.sort = field.sort; this.filter = field.filter; @@ -268,6 +315,15 @@ private Builder(final Column field) { * @return The {@link Builder}, enabling method chaining */ public Builder id(final String value) { + this.id = GwtNullSafe.get(value, CIKey::of); + return this; + } + + /** + * @param value The internal id of the field for equality purposes + * @return The {@link Builder}, enabling method chaining + */ + public Builder id(final CIKey value) { this.id = value; return this; } @@ -277,6 +333,15 @@ public Builder id(final String value) { * @return The {@link Builder}, enabling method chaining */ public Builder name(final String value) { + this.name = GwtNullSafe.get(value, CIKey::of); + return this; + } + + /** + * @param value The name of the field for display purposes + * @return The {@link Builder}, enabling method chaining + */ + public Builder name(final CIKey value) { this.name = value; return this; } diff --git a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ExpressionUtil.java b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ExpressionUtil.java index f7ea702a9c9..2ea08d6d15a 100644 --- a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ExpressionUtil.java +++ b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ExpressionUtil.java @@ -1,9 +1,27 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.api.v2; import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; import stroom.query.api.v2.ExpressionOperator.Op; import stroom.query.api.v2.ExpressionTerm.Condition; +import stroom.util.shared.GwtNullSafe; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.Collection; @@ -105,19 +123,24 @@ public static int termCount(final ExpressionOperator expressionOperator, } public static List fields(final ExpressionOperator expressionOperator) { - return terms(expressionOperator, - null).stream().map(ExpressionTerm::getField).collect(Collectors.toList()); + return terms(expressionOperator, null) + .stream() + .map(ExpressionTerm::getField) + .collect(Collectors.toList()); } public static List values(final ExpressionOperator expressionOperator) { - return terms(expressionOperator, - null).stream().map(ExpressionTerm::getValue).collect(Collectors.toList()); + return terms(expressionOperator, null) + .stream() + .map(ExpressionTerm::getValue) + .collect(Collectors.toList()); } public static List values(final ExpressionOperator expressionOperator, final String fieldName) { - return terms(expressionOperator, - Collections.singleton(fieldName)).stream().map(ExpressionTerm::getValue).collect( - Collectors.toList()); + return terms(expressionOperator, Collections.singleton(fieldName)) + .stream() + .map(ExpressionTerm::getValue).collect( + Collectors.toList()); } public static List terms(final ExpressionOperator expressionOperator, @@ -137,16 +160,16 @@ private static void addTerms(final ExpressionOperator expressionOperator, for (final ExpressionItem item : expressionOperator.getChildren()) { if (item.enabled()) { if (item instanceof ExpressionTerm) { - final ExpressionTerm expressionTerm = (ExpressionTerm) item; + //noinspection PatternVariableCanBeUsed // GWT + final ExpressionTerm term = (ExpressionTerm) item; if (fieldNames == null || fieldNames.stream() .anyMatch(fieldName -> - fieldName.equals(expressionTerm.getField()) && - (Condition.IS_DOC_REF.equals(expressionTerm.getCondition()) && - expressionTerm.getDocRef() != null && - expressionTerm.getDocRef().getUuid() != null) || - (expressionTerm.getValue() != null && - expressionTerm.getValue().length() > 0))) { - terms.add(expressionTerm); + fieldName.equalsIgnoreCase(term.getField()) && + (Condition.IS_DOC_REF.equals(term.getCondition()) && + term.getDocRef() != null && + term.getDocRef().getUuid() != null) || + (GwtNullSafe.isNonEmptyString(term.getValue())))) { + terms.add(term); } } else if (item instanceof ExpressionOperator) { addTerms((ExpressionOperator) item, fieldNames, terms); @@ -207,7 +230,7 @@ public static Query replaceExpressionParameters(final Query query) { if (query != null) { ExpressionOperator expression = query.getExpression(); if (query.getParams() != null && expression != null) { - final Map paramMap = ParamUtil.createParamMap(query.getParams()); + final Map paramMap = ParamUtil.createParamMap(query.getParams()); expression = replaceExpressionParameters(expression, paramMap); } result = query.copy().expression(expression).build(); @@ -219,7 +242,7 @@ public static ExpressionOperator replaceExpressionParameters(final ExpressionOpe final List params) { final ExpressionOperator result; if (operator != null) { - final Map paramMap = ParamUtil.createParamMap(params); + final Map paramMap = ParamUtil.createParamMap(params); result = replaceExpressionParameters(operator, paramMap); } else { result = null; @@ -228,7 +251,7 @@ public static ExpressionOperator replaceExpressionParameters(final ExpressionOpe } public static ExpressionOperator replaceExpressionParameters(final ExpressionOperator operator, - final Map paramMap) { + final Map paramMap) { final ExpressionOperator.Builder builder = ExpressionOperator .builder() .enabled(operator.getEnabled()) @@ -236,10 +259,12 @@ public static ExpressionOperator replaceExpressionParameters(final ExpressionOpe if (operator.getChildren() != null) { for (ExpressionItem child : operator.getChildren()) { if (child instanceof ExpressionOperator) { + //noinspection PatternVariableCanBeUsed // GWT final ExpressionOperator childOperator = (ExpressionOperator) child; builder.addOperator(replaceExpressionParameters(childOperator, paramMap)); } else if (child instanceof ExpressionTerm) { + //noinspection PatternVariableCanBeUsed // GWT final ExpressionTerm term = (ExpressionTerm) child; final String value = term.getValue(); final String replaced = ParamUtil.replaceParameters(value, paramMap); @@ -280,6 +305,7 @@ public static boolean walkExpressionTree(final ExpressionItem expressionItem, } if (continueWalking) { if (expressionItem instanceof ExpressionOperator) { + //noinspection PatternVariableCanBeUsed // GWT final ExpressionOperator expressionOperator = (ExpressionOperator) expressionItem; final List children = expressionOperator.getChildren(); if (children != null && !children.isEmpty()) { @@ -365,6 +391,10 @@ private static ExpressionItem simplifyExpressionItem(final ExpressionItem item) return item; } + + // -------------------------------------------------------------------------------- + + public interface ExpressionItemVisitor { /** diff --git a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/HoppingWindow.java b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/HoppingWindow.java index c2246789d4d..8b87930fbd1 100644 --- a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/HoppingWindow.java +++ b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/HoppingWindow.java @@ -1,5 +1,23 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.api.v2; +import stroom.util.shared.string.CIKeys; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; @@ -86,9 +104,14 @@ public static Builder builder() { return new Builder(); } + + // -------------------------------------------------------------------------------- + + public static class Builder { - private String timeField = "EventTime"; + // private CIKey timeField = CIKeys.EVENT_TIME; + private String timeField = CIKeys.EVENT_TIME.get(); private String windowSize = "10m"; private String advanceSize = "1m"; private String function = "count()"; diff --git a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Param.java b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Param.java index 37ab3d1d9b4..7df9215a7ba 100644 --- a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Param.java +++ b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Param.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ParamSubstituteUtil.java b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ParamSubstituteUtil.java index eac1becb64e..27d093e005b 100644 --- a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ParamSubstituteUtil.java +++ b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ParamSubstituteUtil.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,6 +16,8 @@ package stroom.query.api.v2; +import stroom.util.shared.string.CIKey; + import java.util.ArrayList; import java.util.List; @@ -64,4 +66,8 @@ public static List getParams(final String string) { public static String makeParam(final String param) { return "${" + param + "}"; } + + public static String makeParam(final CIKey param) { + return "${" + param.get() + "}"; + } } diff --git a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ParamUtil.java b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ParamUtil.java index fbf10cc8439..4e5cb19b66b 100644 --- a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ParamUtil.java +++ b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/ParamUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2024 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,11 +17,14 @@ package stroom.query.api.v2; +import stroom.util.shared.GwtNullSafe; +import stroom.util.shared.string.CIKey; + import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public final class ParamUtil { @@ -104,7 +107,8 @@ public static List parse(final String string) { return list; } - public static String replaceParameters(final String value, final Map paramMap) { + public static String replaceParameters(final String value, + final Map paramMap) { if (value == null) { return null; } @@ -152,7 +156,7 @@ public static String replaceParameters(final String value, final Map createParamMap(final List params) { + public static Map createParamMap(final List params) { // Create a parameter map. - final Map paramMap; - if (params != null) { - paramMap = new HashMap<>(); - for (final Param param : params) { - if (param.getKey() != null && param.getValue() != null) { - paramMap.put(param.getKey(), param.getValue()); - } - } - } else { - paramMap = Collections.emptyMap(); - } - return paramMap; + return GwtNullSafe.stream(params) + .filter(param -> + param.getKey() != null + && param.getValue() != null) + .collect(Collectors.toMap( + param -> CIKey.of(param.getKey()), + Param::getValue)); } public static String getCombinedParameterString(final List params) { diff --git a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Sort.java b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Sort.java index 8afe51e8d4e..41eb425b919 100644 --- a/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Sort.java +++ b/stroom-query/stroom-query-api/src/main/java/stroom/query/api/v2/Sort.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,6 +17,7 @@ package stroom.query.api.v2; import stroom.docref.HasDisplayValue; +import stroom.util.shared.GwtNullSafe; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; @@ -26,6 +27,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import java.util.Objects; +import java.util.Optional; @JsonPropertyOrder({"order", "direction"}) @JsonInclude(Include.NON_NULL) @@ -86,30 +88,88 @@ public String toString() { '}'; } + public static Builder builder() { + return new Builder(); + } + + public Builder copy() { + return new Builder(this); + } + + + // -------------------------------------------------------------------------------- + + public enum SortDirection implements HasDisplayValue { - ASCENDING("Ascending"), - DESCENDING("Descending"); + ASCENDING("Ascending", "asc"), + DESCENDING("Descending", "desc"), + ; - private final String displayValue; + private final String longForm; + private final String shortForm; - SortDirection(final String displayValue) { - this.displayValue = displayValue; + SortDirection(final String longForm, String shortForm) { + this.longForm = longForm; + this.shortForm = shortForm; } + /** + * 'Ascending' or 'Descending' + * Equivalent to calling {@link SortDirection#getLongForm()} + */ @Override public String getDisplayValue() { - return displayValue; + return longForm; } - } - public static Builder builder() { - return new Builder(); - } + /** + * 'Ascending' or 'Descending' + */ + public String getLongForm() { + return longForm; + } - public Builder copy() { - return new Builder(this); + /** + * 'asc' or 'desc' + */ + public String getShortForm() { + return shortForm; + } + + /** + * Parse a {@link SortDirection} from a short form string ('asc' or 'desc') ignoring case. + */ + public static Optional fromString(final String str) { + if (!GwtNullSafe.isBlankString(str)) { + for (final SortDirection sortDirection : SortDirection.values()) { + if (sortDirection.shortForm.equalsIgnoreCase(str) + || sortDirection.longForm.equalsIgnoreCase(str)) { + return Optional.of(sortDirection); + } + } + } + return Optional.empty(); + } + + /** + * Parse a {@link SortDirection} from a short form string ('asc' or 'desc') ignoring case. + */ + public static Optional fromShortForm(final String str) { + if (!GwtNullSafe.isBlankString(str)) { + for (final SortDirection sortDirection : SortDirection.values()) { + if (sortDirection.shortForm.equalsIgnoreCase(str)) { + return Optional.of(sortDirection); + } + } + } + return Optional.empty(); + } } + + // -------------------------------------------------------------------------------- + + /** * Builder for constructing a {@link Sort sort} */ diff --git a/stroom-query/stroom-query-api/src/test/java/stroom/query/api/v2/TestSortDirection.java b/stroom-query/stroom-query-api/src/test/java/stroom/query/api/v2/TestSortDirection.java new file mode 100644 index 00000000000..2ea58c7b4e4 --- /dev/null +++ b/stroom-query/stroom-query-api/src/test/java/stroom/query/api/v2/TestSortDirection.java @@ -0,0 +1,84 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.query.api.v2; + +import stroom.query.api.v2.Sort.SortDirection; +import stroom.test.common.TestUtil; + +import com.google.inject.TypeLiteral; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; + +import java.util.Optional; +import java.util.stream.Stream; + +public class TestSortDirection { + + @TestFactory + Stream testFromString() { + return TestUtil.buildDynamicTestStream() + .withInputType(String.class) + .withWrappedOutputType(new TypeLiteral>() { + }) + .withSingleArgTestFunction(SortDirection::fromString) + .withSimpleEqualityAssertion() + .addCase(null, Optional.empty()) + .addCase("", Optional.empty()) + .addCase(" ", Optional.empty()) + .addCase("foo", Optional.empty()) + .addCase("asc", Optional.of(SortDirection.ASCENDING)) + .addCase("Asc", Optional.of(SortDirection.ASCENDING)) + .addCase("ASC", Optional.of(SortDirection.ASCENDING)) + .addCase("ascending", Optional.of(SortDirection.ASCENDING)) + .addCase("Ascending", Optional.of(SortDirection.ASCENDING)) + .addCase("ASCENDING", Optional.of(SortDirection.ASCENDING)) + .addCase("desc", Optional.of(SortDirection.DESCENDING)) + .addCase("Desc", Optional.of(SortDirection.DESCENDING)) + .addCase("DESC", Optional.of(SortDirection.DESCENDING)) + .addCase("descending", Optional.of(SortDirection.DESCENDING)) + .addCase("Descending", Optional.of(SortDirection.DESCENDING)) + .addCase("DESCENDING", Optional.of(SortDirection.DESCENDING)) + .build(); + } + + @TestFactory + Stream testFromShortForm() { + return TestUtil.buildDynamicTestStream() + .withInputType(String.class) + .withWrappedOutputType(new TypeLiteral>() { + }) + .withSingleArgTestFunction(SortDirection::fromShortForm) + .withSimpleEqualityAssertion() + .addCase(null, Optional.empty()) + .addCase("", Optional.empty()) + .addCase(" ", Optional.empty()) + .addCase("foo", Optional.empty()) + .addCase("asc", Optional.of(SortDirection.ASCENDING)) + .addCase("Asc", Optional.of(SortDirection.ASCENDING)) + .addCase("ASC", Optional.of(SortDirection.ASCENDING)) + .addCase("ascending", Optional.empty()) + .addCase("Ascending", Optional.empty()) + .addCase("ASCENDING", Optional.empty()) + .addCase("desc", Optional.of(SortDirection.DESCENDING)) + .addCase("Desc", Optional.of(SortDirection.DESCENDING)) + .addCase("DESC", Optional.of(SortDirection.DESCENDING)) + .addCase("descending", Optional.empty()) + .addCase("Descending", Optional.empty()) + .addCase("DESCENDING", Optional.empty()) + .build(); + } +} diff --git a/stroom-query/stroom-query-api/src/test/resources/searchRequestPortrait-current.txt b/stroom-query/stroom-query-api/src/test/resources/searchRequestPortrait-current.txt index 0a722d01833..808a4c05e44 100644 --- a/stroom-query/stroom-query-api/src/test/resources/searchRequestPortrait-current.txt +++ b/stroom-query/stroom-query-api/src/test/resources/searchRequestPortrait-current.txt @@ -138,6 +138,8 @@ class stroom.query.api.v2.Column - public stroom.query.api.v2.Column$Builder str class stroom.query.api.v2.Column - public stroom.query.api.v2.Filter stroom.query.api.v2.Column.getFilter() class stroom.query.api.v2.Column - public stroom.query.api.v2.Format stroom.query.api.v2.Column.getFormat() class stroom.query.api.v2.Column - public stroom.query.api.v2.Sort stroom.query.api.v2.Column.getSort() +class stroom.query.api.v2.Column - public stroom.util.shared.string.CIKey stroom.query.api.v2.Column.getIdAsCIKey() +class stroom.query.api.v2.Column - public stroom.util.shared.string.CIKey stroom.query.api.v2.Column.getNameAsCIKey() class stroom.query.api.v2.Column$Builder - public boolean java.lang.Object.equals(java.lang.Object) class stroom.query.api.v2.Column$Builder - public final native java.lang.Class java.lang.Object.getClass() class stroom.query.api.v2.Column$Builder - public final native void java.lang.Object.notify() diff --git a/stroom-query/stroom-query-api/src/test/resources/searchRequestPortrait-new.txt b/stroom-query/stroom-query-api/src/test/resources/searchRequestPortrait-new.txt index 0a722d01833..808a4c05e44 100644 --- a/stroom-query/stroom-query-api/src/test/resources/searchRequestPortrait-new.txt +++ b/stroom-query/stroom-query-api/src/test/resources/searchRequestPortrait-new.txt @@ -138,6 +138,8 @@ class stroom.query.api.v2.Column - public stroom.query.api.v2.Column$Builder str class stroom.query.api.v2.Column - public stroom.query.api.v2.Filter stroom.query.api.v2.Column.getFilter() class stroom.query.api.v2.Column - public stroom.query.api.v2.Format stroom.query.api.v2.Column.getFormat() class stroom.query.api.v2.Column - public stroom.query.api.v2.Sort stroom.query.api.v2.Column.getSort() +class stroom.query.api.v2.Column - public stroom.util.shared.string.CIKey stroom.query.api.v2.Column.getIdAsCIKey() +class stroom.query.api.v2.Column - public stroom.util.shared.string.CIKey stroom.query.api.v2.Column.getNameAsCIKey() class stroom.query.api.v2.Column$Builder - public boolean java.lang.Object.equals(java.lang.Object) class stroom.query.api.v2.Column$Builder - public final native java.lang.Class java.lang.Object.getClass() class stroom.query.api.v2.Column$Builder - public final native void java.lang.Object.notify() diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledColumns.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledColumns.java index 1244e466be7..f966b062736 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledColumns.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledColumns.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -26,6 +26,7 @@ import stroom.query.language.functions.ParamFactory; import stroom.query.language.functions.ref.ValueReferenceIndex; import stroom.util.NullSafe; +import stroom.util.shared.string.CIKey; import java.text.ParseException; import java.util.Collections; @@ -52,17 +53,21 @@ private CompiledColumns(final List columns, public static CompiledColumns create(final ExpressionContext expressionContext, final List columns, - final Map paramMap) { + final Map paramMap) { return create(expressionContext, columns, new FieldIndex(), paramMap); } public static CompiledColumns create(final ExpressionContext expressionContext, final List columns, final FieldIndex fieldIndex, - final Map paramMap) { + final Map paramMap) { final ValueReferenceIndex valueReferenceIndex = new ValueReferenceIndex(); if (columns == null) { - return new CompiledColumns(Collections.emptyList(), new CompiledColumn[0], fieldIndex, valueReferenceIndex); + return new CompiledColumns( + Collections.emptyList(), + new CompiledColumn[0], + fieldIndex, + valueReferenceIndex); } final ExpressionParser expressionParser = new ExpressionParser(new ParamFactory(new HashMap<>())); diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledIncludeExcludeFilter.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledIncludeExcludeFilter.java index 2995cb4a0f0..5355720d77e 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledIncludeExcludeFilter.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledIncludeExcludeFilter.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,6 +17,7 @@ package stroom.query.common.v2; import stroom.query.api.v2.Filter; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.List; @@ -27,7 +28,8 @@ public class CompiledIncludeExcludeFilter { - public static Optional> create(final Filter filter, final Map paramMap) { + public static Optional> create(final Filter filter, + final Map paramMap) { if (filter == null) { return Optional.empty(); } @@ -64,7 +66,7 @@ public static Optional> create(final Filter filter, final Map< return optional; } - private static List createPatternList(final String patterns, final Map paramMap) { + private static List createPatternList(final String patterns, final Map paramMap) { List patternList = null; if (patterns != null && !patterns.trim().isEmpty()) { final String replaced = KVMapUtil.replaceParameters(patterns, paramMap); diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledWindow.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledWindow.java index c4edf419402..4120243fbfe 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledWindow.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CompiledWindow.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; import stroom.query.api.v2.Column; @@ -55,6 +71,7 @@ public CompiledWindow(final String timeField, public static CompiledWindow create(final Window window) { SimpleDuration windowSize = SimpleDuration.ZERO; List offsets = Collections.emptyList(); +// CIKey timeField = CIKeys.EVENT_TIME; String timeField = "EventTime"; String function = "count()"; @@ -185,12 +202,20 @@ public WindowProcessor createWindowProcessor(final FieldIndex fieldIndex) { return new OffsetWindowProcessor(windowSize, offsets, windowTimeFieldPos); } + + // -------------------------------------------------------------------------------- + + public interface WindowProcessor { void process(Val[] values, BiConsumer consumer); } + + // -------------------------------------------------------------------------------- + + private static class NoOpWindowProcessor implements WindowProcessor { @Override @@ -200,6 +225,10 @@ public void process(final Val[] values, } } + + // -------------------------------------------------------------------------------- + + private static class OffsetWindowProcessor implements WindowProcessor { private final SimpleDuration windowSize; diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/ConditionalFormattingRowCreator.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/ConditionalFormattingRowCreator.java index 74f11f8d856..737400c01fa 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/ConditionalFormattingRowCreator.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/ConditionalFormattingRowCreator.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; import stroom.expression.api.DateTimeSettings; @@ -9,8 +25,7 @@ import stroom.query.language.functions.ref.ErrorConsumer; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; - -import com.google.common.base.Predicates; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.List; @@ -22,7 +37,7 @@ public class ConditionalFormattingRowCreator extends FilteredRowCreator { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(ConditionalFormattingRowCreator.class); - private final Predicate> rowFilter; + private final Predicate> rowFilter; private final List rules; private final ErrorConsumer errorConsumer; @@ -30,7 +45,7 @@ private ConditionalFormattingRowCreator(final List originalColumns, final List newColumns, final ColumnFormatter columnFormatter, final KeyFactory keyFactory, - final Predicate> rowFilter, + final Predicate> rowFilter, final List rules, final ErrorConsumer errorConsumer) { super(originalColumns, newColumns, columnFormatter, keyFactory, rowFilter, errorConsumer); @@ -57,9 +72,9 @@ public static Optional> create(final List originalColumn if (!activeRules.isEmpty()) { final Optional optionalRowExpressionMatcher = RowExpressionMatcher.create(newColumns, dateTimeSettings, rowFilterExpression); - final Predicate> rowFilter = optionalRowExpressionMatcher - .map(orem -> (Predicate>) orem) - .orElse(Predicates.alwaysTrue()); + final Predicate> rowFilter = optionalRowExpressionMatcher + .map(orem -> (Predicate>) orem) + .orElse(RowExpressionMatcher.ALWAYS_TRUE_PREDICATE); final List ruleAndMatchers = new ArrayList<>(); for (final ConditionalFormattingRule rule : rules) { @@ -88,9 +103,22 @@ public static Optional> create(final List originalColumn @Override public Row create(final Item item, final List stringValues, - final Map fieldIdToValueMap) { + final Map fieldIdToValueMap) { Row row = null; + // Removed in 7.5 +// final Map fieldIdToValueMap = new HashMap<>(); +// final List stringValues = new ArrayList<>(columns.size()); +// int i = 0; +// for (final Column column : columns) { +// final Val val = item.getValue(i); +// final String string = fieldFormatter.format(column, val); +// stringValues.add(string); +// fieldIdToValueMap.put(column.getId(), string); +// fieldIdToValueMap.put(column.getName(), string); +// i++; +// } + // Find a matching rule. ConditionalFormattingRule matchingRule = null; diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CoprocessorsFactory.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CoprocessorsFactory.java index e3034cee345..12175dff8b1 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CoprocessorsFactory.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CoprocessorsFactory.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; import stroom.docref.DocRef; @@ -15,6 +31,7 @@ import stroom.util.NullSafe; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; @@ -102,7 +119,7 @@ public CoprocessorsImpl create(final SearchRequestSource searchRequestSource, final FieldIndex fieldIndex = new FieldIndex(); // Create a parameter map. - final Map paramMap = ParamUtil.createParamMap(params); + final Map paramMap = ParamUtil.createParamMap(params); // Create error consumer. final ErrorConsumer errorConsumer = new ErrorConsumerImpl(); @@ -165,7 +182,7 @@ private Coprocessor create(final ExpressionContext expressionContext, final QueryKey queryKey, final CoprocessorSettings settings, final FieldIndex fieldIndex, - final Map paramMap, + final Map paramMap, final ErrorConsumer errorConsumer, final DataStoreSettings dataStoreSettings) { if (settings instanceof final TableCoprocessorSettings tableCoprocessorSettings) { @@ -194,7 +211,7 @@ private DataStore create(final ExpressionContext expressionContext, final String componentId, final TableSettings tableSettings, final FieldIndex fieldIndex, - final Map paramMap, + final Map paramMap, final DataStoreSettings dataStoreSettings, final ErrorConsumer errorConsumer) { diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CoprocessorsImpl.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CoprocessorsImpl.java index d42cee52e41..9feaa648eaf 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CoprocessorsImpl.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/CoprocessorsImpl.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; import stroom.docref.DocRef; @@ -153,7 +169,7 @@ public DataStore getData(final String componentId) { @Override public boolean isPresent() { - return coprocessorMap.size() > 0; + return !coprocessorMap.isEmpty(); } public long getValueCount() { diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/DataStoreFactory.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/DataStoreFactory.java index c0a0fd785c1..ca68696d653 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/DataStoreFactory.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/DataStoreFactory.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; import stroom.query.api.v2.QueryKey; @@ -6,6 +22,7 @@ import stroom.query.language.functions.ExpressionContext; import stroom.query.language.functions.FieldIndex; import stroom.query.language.functions.ref.ErrorConsumer; +import stroom.util.shared.string.CIKey; import java.util.Map; @@ -18,12 +35,16 @@ DataStore create( String componentId, TableSettings tableSettings, FieldIndex fieldIndex, - Map paramMap, + Map paramMap, DataStoreSettings dataStoreSettings, ErrorConsumer errorConsumer); StoreSizeSummary getTotalSizeOnDisk(); + + // -------------------------------------------------------------------------------- + + class StoreSizeSummary { private final long totalSizeOnDisk; diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/DateExpressionParser.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/DateExpressionParser.java index c715590afcb..843784b379c 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/DateExpressionParser.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/DateExpressionParser.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,6 +16,7 @@ package stroom.query.common.v2; +import stroom.datasource.api.v2.QueryField; import stroom.expression.api.DateTimeSettings; import stroom.query.api.v2.TimeFilter; import stroom.query.api.v2.TimeRange; @@ -74,6 +75,11 @@ private static long getMs(final String expression, .orElse(defaultValue); } + public static long getMs(final QueryField field, final String value) { + Objects.requireNonNull(field); + return getMs(field.getFldName(), value, DateTimeSettings.builder().build()); + } + public static long getMs(final String fieldName, final String value) { return getMs(fieldName, value, DateTimeSettings.builder().build()); } diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/EventCoprocessor.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/EventCoprocessor.java index 48ee848c446..31a73a90fef 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/EventCoprocessor.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/EventCoprocessor.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,8 @@ import stroom.query.language.functions.FieldIndex; import stroom.query.language.functions.Val; import stroom.query.language.functions.ref.ErrorConsumer; +import stroom.util.shared.query.FieldNames; +import stroom.util.shared.string.CIKey; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; @@ -28,8 +30,8 @@ public class EventCoprocessor implements Coprocessor, HasCompletionState { - private static final String STREAM_ID = "StreamId"; - private static final String EVENT_ID = "EventId"; + private static final CIKey STREAM_ID = FieldNames.FALLBACK_STREAM_ID_FIELD_KEY; + private static final CIKey EVENT_ID = FieldNames.FALLBACK_EVENT_ID_FIELD_KEY; private final ErrorConsumer errorConsumer; private final EventRef minEvent; @@ -54,11 +56,8 @@ public EventCoprocessor(final EventCoprocessorSettings settings, this.maxEventsPerStream = settings.getMaxEventsPerStream(); // Add required fields. - fieldIndex.create(STREAM_ID); - fieldIndex.create(EVENT_ID); - - streamIdIndex = fieldIndex.getPos(STREAM_ID); - eventIdIndex = fieldIndex.getPos(EVENT_ID); + streamIdIndex = fieldIndex.create(STREAM_ID); + eventIdIndex = fieldIndex.create(EVENT_ID); } @Override diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/FilteredRowCreator.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/FilteredRowCreator.java index e9b997a418f..b697d355f06 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/FilteredRowCreator.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/FilteredRowCreator.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; import stroom.expression.api.DateTimeSettings; @@ -9,8 +25,7 @@ import stroom.query.language.functions.ref.ErrorConsumer; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; - -import com.google.common.base.Predicates; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.HashMap; @@ -23,14 +38,14 @@ public class FilteredRowCreator extends SimpleRowCreator { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(FilteredRowCreator.class); - private final Predicate> rowFilter; + private final Predicate> rowFilter; FilteredRowCreator(final List originalColumns, - final List newColumns, - final ColumnFormatter columnFormatter, - final KeyFactory keyFactory, - final Predicate> rowFilter, - final ErrorConsumer errorConsumer) { + final List newColumns, + final ColumnFormatter columnFormatter, + final KeyFactory keyFactory, + final Predicate> rowFilter, + final ErrorConsumer errorConsumer) { super(originalColumns, newColumns, columnFormatter, keyFactory, errorConsumer); this.rowFilter = rowFilter; @@ -44,11 +59,11 @@ public static Optional> create(final List originalColumn final DateTimeSettings dateTimeSettings, final ErrorConsumer errorConsumer) { if (ExpressionUtil.hasTerms(rowFilterExpression)) { - final Optional optionalRowExpressionMatcher = + final Optional optRowExpressionMatcher = RowExpressionMatcher.create(newColumns, dateTimeSettings, rowFilterExpression); - final Predicate> rowFilter = optionalRowExpressionMatcher - .map(orem -> (Predicate>) orem) - .orElse(Predicates.alwaysTrue()); + final Predicate> rowFilter = optRowExpressionMatcher + .map(orem -> (Predicate>) orem) + .orElse(RowExpressionMatcher.ALWAYS_TRUE_PREDICATE); return Optional.of(new FilteredRowCreator( originalColumns, @@ -63,13 +78,13 @@ public static Optional> create(final List originalColumn @Override public final Row create(final Item item) { - final Map fieldIdToValueMap = new HashMap<>(); + final Map fieldIdToValueMap = new HashMap<>(); final List stringValues = new ArrayList<>(functions.size()); functions.forEach(f -> { final String string = f.apply(item); stringValues.add(string); - fieldIdToValueMap.put(f.column.getId(), string); - fieldIdToValueMap.put(f.column.getName(), string); + fieldIdToValueMap.put(f.column.getIdAsCIKey(), string); + fieldIdToValueMap.put(f.column.getNameAsCIKey(), string); }); return create(item, stringValues, fieldIdToValueMap); @@ -77,7 +92,7 @@ public final Row create(final Item item) { public Row create(final Item item, final List stringValues, - final Map fieldIdToValueMap) { + final Map fieldIdToValueMap) { Row row = null; try { // See if we can exit early by applying row filter. diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/FlatResultCreator.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/FlatResultCreator.java index 71965520a75..d78312408f4 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/FlatResultCreator.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/FlatResultCreator.java @@ -40,6 +40,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.Collections; @@ -64,7 +65,7 @@ public FlatResultCreator(final DataStoreFactory dataStoreFactory, final String componentId, final ExpressionContext expressionContext, final ResultRequest resultRequest, - final Map paramMap, + final Map paramMap, final ColumnFormatter columnFormatter, final Sizes defaultMaxResultsSizes, final boolean cacheLastResult) { @@ -329,7 +330,7 @@ private static class Mapper { private final QueryKey queryKey; private final String componentId; private final TableSettings child; - private final Map paramMap; + private final Map paramMap; private final ErrorConsumer errorConsumer; private final FieldIndex childFieldIndex; @@ -343,7 +344,7 @@ private static class Mapper { final String componentId, final TableSettings parent, final TableSettings child, - final Map paramMap, + final Map paramMap, final ErrorConsumer errorConsumer) { this.dataStoreFactory = dataStoreFactory; this.dataStoreSettings = dataStoreSettings; @@ -359,7 +360,7 @@ private static class Mapper { // Parent fields are now table column names. for (final Column column : parent.getColumns()) { - parentFieldIndex.create(column.getName()); + parentFieldIndex.create(column.getNameAsCIKey()); } // Extract child fields from expressions. diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldMap.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldMap.java new file mode 100644 index 00000000000..b8397cbacae --- /dev/null +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldMap.java @@ -0,0 +1,147 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.query.common.v2; + +import stroom.datasource.api.v2.IndexField; +import stroom.query.common.v2.IndexFieldMapImpl.EmptyIndexFieldMap; +import stroom.query.common.v2.IndexFieldMapImpl.SingleIndexField; +import stroom.util.NullSafe; +import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; +import stroom.util.string.MultiCaseMap.MultipleMatchException; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Holds one or more {@link IndexField}s that ALL share the same name when ignoring case. + * Provides a common means to get a field for a given field name, initially ignoring case, + * but taking case into account if the map contains multiple fields (with the same + * case-insensitive name). + *

+ * E.g. it may hold fields 'foo' and 'FOO'. Calling {@link IndexFieldMap#getClosestMatchingField(String)} + * with 'foo' would return field 'foo', but calling that with 'Foo' would throw an exception as there + * is no exact match. + *

+ */ +public interface IndexFieldMap { + + /** + * @param fieldName A case-insensitive field name. The string it wraps may or may not exactly match + * one of the field names in the map, but it must match all when ignoring case. + * @param fieldNameToFieldMap A map of case-sensitive field names to their respective {@link IndexField} object. + * All keys must be equal ignoring case to each other and to fieldName. + */ + static IndexFieldMap fromFieldsMap(final CIKey fieldName, + final Map fieldNameToFieldMap) { + final int size = NullSafe.size(fieldNameToFieldMap); + if (size == 0) { + return new EmptyIndexFieldMap(fieldName); + } else if (size == 1) { + return IndexFieldMap.forSingleField(fieldName, fieldNameToFieldMap.values().iterator().next()); + } else { + return new IndexFieldMapImpl(fieldName, fieldNameToFieldMap); + } + } + + /** + * @param fieldName A case-insensitive field name. The string it wraps may or may not exactly match + * one of the field names in the map, but it must match all when ignoring case. + * @param indexFields A collection of {@link IndexField} whose field names are ALL equal (ignoring case). + */ + static IndexFieldMap fromFieldList(final CIKey fieldName, + final Collection indexFields) { + final int size = NullSafe.size(indexFields); + if (size == 0) { + return new EmptyIndexFieldMap(fieldName); + } else if (size == 1) { + return IndexFieldMap.forSingleField(fieldName, indexFields.iterator().next()); + } else { + return new IndexFieldMapImpl(fieldName, indexFields.stream() + .collect(Collectors.toMap(IndexField::getFldName, Function.identity()))); + } + } + + /** + * Constructor for use when you know there is only one {@link IndexField} corresponding + * to fieldName (ignoring case). + */ + static IndexFieldMap forSingleField(final CIKey fieldName, + final IndexField indexField) { + if (indexField == null) { + return new EmptyIndexFieldMap(fieldName); + } else { + return new SingleIndexField(fieldName, indexField); + } + } + + static IndexFieldMap empty(final CIKey fieldName) { + return new EmptyIndexFieldMap(fieldName); + } + + static IndexFieldMap merge(final IndexFieldMap map1, final IndexFieldMap map2) { + if (map1 == null && map2 == null) { + return null; + } else if (map1 == null) { + return map2; + } else if (map2 == null) { + return map1; + } else { + if (!Objects.equals(map1.getFieldName(), map2.getFieldName())) { + throw new IllegalArgumentException(LogUtil.message( + "Names do not match. map1: '{}', map2: '{}'", + map1.getFieldName(), map2.getFieldName())); + } + + final Map combinedMap = Stream.of(map1, map2) + .map(IndexFieldMap::getFields) + .flatMap(List::stream) + .collect(Collectors.toMap( + IndexField::getFldName, Function.identity())); + return IndexFieldMap.fromFieldsMap(map1.getFieldName(), combinedMap); + } + } + + /** + * @return The case-insensitive field name. + */ + CIKey getFieldName(); + + /** + * @return The list of {@link IndexField} matching the case-insensitive {@link CIKey} fieldName. + */ + List getFields(); + + /** + * @return The {@link IndexField} associated with ciKey (case-insensitive) or null if there + * is no associated value. If ciKey matches multiple keys with different case then + * it will attempt to do a case-sensitive match. If there is no case-sensitive match + * it will throw a {@link MultipleMatchException}. + * @throws MultipleMatchException if multiple values are associated with ciKey + */ + IndexField getClosestMatchingField(final String caseSensitiveFieldName); + + /** + * @return The {@link IndexField} with a name exactly matching caseSensitiveFieldName, else null. + */ + IndexField getExactMatchingField(final String caseSensitiveFieldName); +} diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldMapImpl.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldMapImpl.java new file mode 100644 index 00000000000..16ed2b518c0 --- /dev/null +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldMapImpl.java @@ -0,0 +1,193 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.query.common.v2; + +import stroom.datasource.api.v2.IndexField; +import stroom.util.NullSafe; +import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; +import stroom.util.string.MultiCaseMap.MultipleMatchException; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Holds a map of case-sensitive field names to their respective {@link IndexField} object. + * ALL field names held in the map are equal when ignoring case. + *

+ * This object acts as the link between case-insensitive field naming in stroom and the potentially + * case-sensitive field naming in the index provider. + *

+ */ +public class IndexFieldMapImpl implements IndexFieldMap { + + private final CIKey caseInsenseFieldName; + private final Map caseSenseFieldNameToFieldMap; + + IndexFieldMapImpl(final CIKey fieldName, + final Map fieldNameToFieldMap) { + Objects.requireNonNull(fieldName); + this.caseInsenseFieldName = fieldName; + this.caseSenseFieldNameToFieldMap = Objects.requireNonNullElseGet( + fieldNameToFieldMap, Collections::emptyMap); + + this.caseSenseFieldNameToFieldMap.keySet() + .forEach(exactFieldName -> { + if (!fieldName.equalsIgnoreCase(exactFieldName)) { + throw new IllegalArgumentException(LogUtil.message( + "fieldName '{}' and exactFieldName '{}' do not match (case insensitive)")); + } + }); + } + + @Override + public CIKey getFieldName() { + return caseInsenseFieldName; + } + + @Override + public List getFields() { + return caseSenseFieldNameToFieldMap.values() + .stream() + .map(indexField -> (IndexField) indexField) + .toList(); + } + + /** + * @return The {@link IndexField} associated with ciKey (case-insensitive) or null if there + * is no associated value. If ciKey matches multiple keys with different case then + * it will attempt to do a case-sensitive match. If there is no case-sensitive match + * it will throw a {@link MultipleMatchException}. + * @throws MultipleMatchException if multiple values are associated with ciKey + */ + @Override + public IndexField getClosestMatchingField(final String caseSensitiveFieldName) { + if (caseInsenseFieldName.equalsIgnoreCase(caseSensitiveFieldName)) { + if (NullSafe.isEmptyMap(caseSenseFieldNameToFieldMap)) { + return null; + } else { + final int count = caseSenseFieldNameToFieldMap.size(); + if (count == 0) { + return null; + } else if (count == 1) { + // There is only one, so we don't need to check for an exact match + return caseSenseFieldNameToFieldMap.values().iterator().next(); + } else { + final IndexField exactMatch = caseSenseFieldNameToFieldMap.get(caseSensitiveFieldName); + if (exactMatch != null) { + return exactMatch; + } else { + throw new MultipleMatchException("Multiple values (" + caseSenseFieldNameToFieldMap.size() + + ") exist for case-insensitive field '" + caseSensitiveFieldName + "'"); + } + } + } + } else { + return null; + } + } + + @Override + public IndexField getExactMatchingField(final String caseSensitiveFieldName) { + return caseSenseFieldNameToFieldMap.get(caseSensitiveFieldName); + } + + + // -------------------------------------------------------------------------------- + + + static class SingleIndexField implements IndexFieldMap { + + private final CIKey fieldName; + private final IndexField indexField; + + public SingleIndexField(final CIKey fieldName, + final IndexField indexField) { + this.fieldName = Objects.requireNonNull(fieldName); + this.indexField = indexField; + + if (indexField != null + && !fieldName.equalsIgnoreCase(indexField.getFldName())) { + throw new IllegalArgumentException(LogUtil.message( + "fieldName '{}' and indexField.fldName '{}' do not match (case insensitive)")); + } + } + + @Override + public CIKey getFieldName() { + return fieldName; + } + + @Override + public List getFields() { + return List.of(indexField); + } + + @Override + public IndexField getClosestMatchingField(final String caseSensitiveFieldName) { + if (fieldName.equalsIgnoreCase(caseSensitiveFieldName)) { + return indexField; + } else { + return null; + } + } + + @Override + public IndexField getExactMatchingField(final String caseSensitiveFieldName) { + if (Objects.equals(indexField.getFldName(), caseSensitiveFieldName)) { + return indexField; + } else { + return null; + } + } + } + + + // -------------------------------------------------------------------------------- + + + static class EmptyIndexFieldMap implements IndexFieldMap { + + private final CIKey fieldName; + + EmptyIndexFieldMap(final CIKey fieldName) { + this.fieldName = fieldName; + } + + @Override + public CIKey getFieldName() { + return fieldName; + } + + @Override + public List getFields() { + return Collections.emptyList(); + } + + @Override + public IndexField getClosestMatchingField(final String caseSensitiveFieldName) { + return null; + } + + @Override + public IndexField getExactMatchingField(final String caseSensitiveFieldName) { + return null; + } + } +} diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldProvider.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldProvider.java index d9f6c6b3d64..7ba29d2feee 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldProvider.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldProvider.java @@ -1,11 +1,34 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; -import stroom.datasource.api.v2.IndexField; import stroom.docref.DocRef; +import stroom.util.shared.string.CIKey; public interface IndexFieldProvider { - IndexField getIndexField(DocRef docRef, String fieldName); + /** + * Get all {@link stroom.datasource.api.v2.IndexField}s matching (ignoring case) the fieldName {@link CIKey}. + * Will return an empty map if no matching fields are found. + */ + IndexFieldMap getIndexFields(DocRef docRef, CIKey fieldName); + /** + * @return The {@link DocRef} type of the {@link IndexFieldProvider} + */ String getType(); } diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldProviders.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldProviders.java index 764dfb00784..14e079a772f 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldProviders.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/IndexFieldProviders.java @@ -1,9 +1,25 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; -import stroom.datasource.api.v2.IndexField; import stroom.docref.DocRef; +import stroom.util.shared.string.CIKey; public interface IndexFieldProviders { - IndexField getIndexField(DocRef docRef, String fieldName); + IndexFieldMap getIndexFields(DocRef docRef, CIKey fieldName); } diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/KVMapUtil.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/KVMapUtil.java index 33cf8587bc3..9ca36ffb63c 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/KVMapUtil.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/KVMapUtil.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,6 +16,9 @@ package stroom.query.common.v2; +import stroom.util.NullSafe; +import stroom.util.shared.string.CIKey; + import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -26,19 +29,15 @@ private KVMapUtil() { // Utility class. } - public static Map parse(final String string) { + public static Map parse(final String string) { // Create a parameter map. - if (string == null) { + if (NullSafe.isBlankString(string)) { return Collections.emptyMap(); } final String trimmed = string.trim(); - if (trimmed.length() == 0) { - return Collections.emptyMap(); - } - - final Map paramMap = new HashMap<>(); + final Map paramMap = new HashMap<>(); final char[] chars = trimmed.toCharArray(); boolean quot = false; @@ -76,9 +75,9 @@ public static Map parse(final String string) { int index = text.lastIndexOf(' '); if (index != -1) { - if (key != null && key.length() > 0) { + if (NullSafe.isNonEmptyString(key)) { final String value = text.substring(0, index).trim(); - paramMap.put(key, value); + paramMap.put(CIKey.of(key), value); } key = text.substring(index + 1).trim(); @@ -94,15 +93,16 @@ public static Map parse(final String string) { } } - if (key != null && key.length() > 0) { + if (NullSafe.isNonEmptyString(key)) { final String value = sb.toString().trim(); - paramMap.put(key, value); + paramMap.put(CIKey.of(key), value); } return paramMap; } - public static String replaceParameters(final String value, final Map paramMap) { + public static String replaceParameters(final String value, + final Map paramMap) { final StringBuilder sb = new StringBuilder(); int paramStart = -1; @@ -146,10 +146,7 @@ public static String replaceParameters(final String value, final Map paramMap, + final Map paramMap, final DataStoreSettings dataStoreSettings, final Provider executorProvider, final ErrorConsumer errorConsumer, @@ -241,6 +243,47 @@ public void accept(final Val[] values) { // Now add the rows if we aren't filtering. windowProcessor.process(values, this::addInternal); } + + + // From merge +// final StoredValues storedValues = valueReferenceIndex.createStoredValues(); +// Map fieldIdToValueMap = null; +// for (final CompiledColumn compiledColumn : compiledColumnArray) { +// final Generator generator = compiledColumn.getGenerator(); +// if (generator != null) { +// final CompiledFilter compiledFilter = compiledColumn.getCompiledFilter(); +// String string = null; +// if (compiledFilter != null || valueFilter != null) { +// generator.set(values, storedValues); +// string = generator.eval(storedValues, null).toString(); +// } +// +// if (compiledFilter != null && !compiledFilter.match(string)) { +// // We want to exclude this item so get out of this method ASAP. +// return; +// } else if (valueFilter != null) { +// if (fieldIdToValueMap == null) { +// fieldIdToValueMap = new HashMap<>(); +// } +// final CIKey caseInsensitiveFieldName = CIKey.of(compiledColumn.getColumn().getName()); +// fieldIdToValueMap.put(caseInsensitiveFieldName, string); +// } +// } +// } +// +// if (fieldIdToValueMap != null) { +// try { +// // If the value filter doesn't match then get out of here now. +// if (!columnExpressionMatcher.match(fieldIdToValueMap, valueFilter)) { +// return; +// } +// } catch (final RuntimeException e) { +// error(e); +// return; +// } +// } + + } private void addInternal(final Val[] values, diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/LmdbDataStoreFactory.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/LmdbDataStoreFactory.java index b20879dd6ab..a0d797235bb 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/LmdbDataStoreFactory.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/LmdbDataStoreFactory.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; import stroom.bytebuffer.impl6.ByteBufferFactory; @@ -17,6 +33,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -74,7 +91,7 @@ public DataStore create(final ExpressionContext expressionContext, final String componentId, final TableSettings tableSettings, final FieldIndex fieldIndex, - final Map paramMap, + final Map paramMap, final DataStoreSettings dataStoreSettings, final ErrorConsumer errorConsumer) { diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/MapDataStore.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/MapDataStore.java index 4b0530b4e46..c473a89907e 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/MapDataStore.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/MapDataStore.java @@ -31,6 +31,7 @@ import stroom.query.language.functions.ref.ValueReferenceIndex; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; @@ -83,7 +84,7 @@ public MapDataStore(final String componentId, final TableSettings tableSettings, final ExpressionContext expressionContext, final FieldIndex fieldIndex, - final Map paramMap, + final Map paramMap, final DataStoreSettings dataStoreSettings, final ErrorConsumer errorConsumer, final ResultStoreMapConfig resultStoreMapConfig) { diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/MapDataStoreFactory.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/MapDataStoreFactory.java index 0b51de8ff3c..3fa38baef66 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/MapDataStoreFactory.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/MapDataStoreFactory.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; import stroom.query.api.v2.QueryKey; @@ -6,6 +22,7 @@ import stroom.query.language.functions.ExpressionContext; import stroom.query.language.functions.FieldIndex; import stroom.query.language.functions.ref.ErrorConsumer; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -28,7 +45,7 @@ public DataStore create(final ExpressionContext expressionContext, final String componentId, final TableSettings tableSettings, final FieldIndex fieldIndex, - final Map paramMap, + final Map paramMap, final DataStoreSettings dataStoreSettings, final ErrorConsumer errorConsumer) { final SearchResultStoreConfig resultStoreConfig = resultStoreConfigProvider.get(); diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/RowExpressionMatcher.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/RowExpressionMatcher.java index 9dd8a093ab5..4c33ad8527c 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/RowExpressionMatcher.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/RowExpressionMatcher.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.query.common.v2; @@ -29,6 +28,7 @@ import stroom.query.language.functions.DateUtil; import stroom.util.NullSafe; import stroom.util.shared.CompareUtil; +import stroom.util.shared.string.CIKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,14 +44,16 @@ import java.util.function.Predicate; import java.util.regex.Pattern; -public class RowExpressionMatcher implements Predicate> { +public class RowExpressionMatcher implements Predicate> { private static final Logger LOGGER = LoggerFactory.getLogger(RowExpressionMatcher.class); private static final String DELIMITER = ","; + public static final Predicate> ALWAYS_TRUE_PREDICATE = var -> true; + private final Map patternCache = new ConcurrentHashMap<>(); - private final Map columnNameToColumnMap; + private final Map columnNameToColumnMap; private final DateTimeSettings dateTimeSettings; private final ExpressionItem item; @@ -62,17 +64,17 @@ public static Optional create(final List columns, return Optional.empty(); } - final Map columnNameToColumnMap = new HashMap<>(); + final Map columnNameToColumnMap = new HashMap<>(); for (final Column column : NullSafe.list(columns)) { // Allow match by id and name. - columnNameToColumnMap.putIfAbsent(column.getId(), column); - columnNameToColumnMap.putIfAbsent(column.getName(), column); + columnNameToColumnMap.putIfAbsent(column.getIdAsCIKey(), column); + columnNameToColumnMap.putIfAbsent(column.getNameAsCIKey(), column); } return Optional.of(new RowExpressionMatcher(columnNameToColumnMap, dateTimeSettings, expression)); } - private RowExpressionMatcher(final Map columnNameToColumnMap, + private RowExpressionMatcher(final Map columnNameToColumnMap, final DateTimeSettings dateTimeSettings, final ExpressionItem item) { this.columnNameToColumnMap = columnNameToColumnMap; @@ -80,34 +82,32 @@ private RowExpressionMatcher(final Map columnNameToColumnMap, this.item = item; } - public boolean isRequiredColumn(final String name) { + public boolean isRequiredColumn(final CIKey name) { return columnNameToColumnMap.containsKey(name); } @Override - public boolean test(final Map attributeMap) { + public boolean test(final Map attributeMap) { return matchItem(attributeMap, item); } - private boolean matchItem(final Map attributeMap, final ExpressionItem item) { + private boolean matchItem(final Map attributeMap, final ExpressionItem item) { if (!item.enabled()) { // If the child item is not enabled then return and keep trying to match with other parts // of the expression. return true; } - if (item instanceof ExpressionOperator) { - return matchOperator(attributeMap, (ExpressionOperator) item); - } else if (item instanceof ExpressionTerm) { - return matchTerm(attributeMap, (ExpressionTerm) item); - } else { - throw new MatchException("Unexpected item type"); - } + return switch (item) { + case ExpressionOperator operator -> matchOperator(attributeMap, operator); + case ExpressionTerm term -> matchTerm(attributeMap, term); + case null, default -> throw new MatchException("Unexpected item type"); + }; } - private boolean matchOperator(final Map attributeMap, + private boolean matchOperator(final Map attributeMap, final ExpressionOperator operator) { - if (operator.getChildren() == null || operator.getChildren().isEmpty()) { + if (NullSafe.isEmptyCollection(operator.getChildren())) { return true; } @@ -133,23 +133,23 @@ private boolean matchOperator(final Map attributeMap, }; } - private boolean matchTerm(final Map attributeMap, final ExpressionTerm term) { + private boolean matchTerm(final Map attributeMap, final ExpressionTerm term) { // The term field is the column name, NOT the index field name final Condition condition = term.getCondition(); // Try and find the referenced field. - String termField = term.getField(); - if (NullSafe.isBlankString(termField)) { + if (NullSafe.isBlankString(term.getField())) { throw new MatchException("Field not set"); } - termField = termField.trim(); - final Column column = columnNameToColumnMap.get(termField); + final String termField = term.getField().trim(); + final CIKey caseInsensitiveTermField = CIKey.of(termField); + final Column column = columnNameToColumnMap.get(caseInsensitiveTermField); if (column == null) { throw new MatchException("Column not found: " + termField); } final String columnName = column.getName(); - final Object attribute = attributeMap.get(termField); + final Object attribute = attributeMap.get(caseInsensitiveTermField); if (Condition.IS_NULL.equals(condition)) { return attribute == null; } else if (Condition.IS_NOT_NULL.equals(condition)) { @@ -159,14 +159,13 @@ private boolean matchTerm(final Map attributeMap, final Expressi } // Try and resolve the term value. - String termValue = term.getValue(); - if (NullSafe.isBlankString(termValue)) { + String termValue = NullSafe.trim(term.getValue()); + if (termValue.isEmpty()) { throw new MatchException("Value not set"); } - termValue = termValue.trim(); // Substitute with row value if a row value exists. - final Object rowValue = attributeMap.get(termValue); + final Object rowValue = attributeMap.get(CIKey.of(termValue)); if (rowValue != null) { termValue = rowValue.toString(); } @@ -364,7 +363,7 @@ private boolean isStringContainsMatch(final String termValue, final Object attri return true; } else if (attribute == null) { return false; - } else if (termValue == null || termValue.isEmpty()) { + } else if (NullSafe.isEmptyString(termValue)) { return true; } else { return isStringMatch(termValue, attribute) || attribute.toString().contains(termValue); @@ -376,12 +375,11 @@ private BigDecimal getNumber(final String columnName, final Object value) { return null; } else { try { - if (value instanceof Long) { - return BigDecimal.valueOf((long) value); - } else if (value instanceof Double) { - return BigDecimal.valueOf((Double) value); - } - return new BigDecimal(value.toString()); + return switch (value) { + case Long aLong -> BigDecimal.valueOf(aLong); + case Double aDouble -> BigDecimal.valueOf(aDouble); + default -> new BigDecimal(value.toString()); + }; } catch (final NumberFormatException e) { throw new MatchException( "Expected a numeric value for field \"" + columnName + @@ -460,6 +458,10 @@ private Long[] getDates(final Object value) { } } + + // -------------------------------------------------------------------------------- + + private static class MatchException extends RuntimeException { MatchException(final String message) { diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/ValFilter.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/ValFilter.java index 9d7a8482a2e..e0ad779792a 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/ValFilter.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/common/v2/ValFilter.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.common.v2; import stroom.expression.api.DateTimeSettings; @@ -7,8 +23,8 @@ import stroom.query.language.functions.ref.StoredValues; import stroom.query.language.functions.ref.ValueReferenceIndex; import stroom.util.NullSafe; - -import com.google.common.base.Predicates; +import stroom.util.PredicateUtil; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.Collections; @@ -23,18 +39,20 @@ public class ValFilter { + private static final Predicate ALWAYS_TRUE_VAL_PREDICATE = valArr -> true; + public static Predicate create(final ExpressionOperator rowExpression, final CompiledColumns compiledColumns, final DateTimeSettings dateTimeSettings, - final Map paramMap, + final Map paramMap, final Consumer throwableConsumer) { - final Optional optionalRowExpressionMatcher = + final Optional optRowExpressionMatcher = RowExpressionMatcher.create(compiledColumns.getColumns(), dateTimeSettings, rowExpression); // See if any of the columns need mapping for the row filter. - final boolean needObjectMap = optionalRowExpressionMatcher.map(rowExpressionMatcher -> { + final boolean needObjectMap = optRowExpressionMatcher.map(rowExpressionMatcher -> { for (final CompiledColumn compiledColumn : compiledColumns.getCompiledColumns()) { - if (rowExpressionMatcher.isRequiredColumn(compiledColumn.getColumn().getName())) { + if (rowExpressionMatcher.isRequiredColumn(compiledColumn.getColumn().getNameAsCIKey())) { return true; } } @@ -43,23 +61,24 @@ public static Predicate create(final ExpressionOperator rowExpression, final List usedColumns = new ArrayList<>(); for (final CompiledColumn compiledColumn : compiledColumns.getCompiledColumns()) { - final boolean needsMapping = optionalRowExpressionMatcher - .map(matcher -> matcher.isRequiredColumn(compiledColumn.getColumn().getName())) + final boolean needsMapping = optRowExpressionMatcher + .map(matcher -> matcher.isRequiredColumn(compiledColumn.getColumn().getNameAsCIKey())) .orElse(false); - final Optional> optionalColumnIncludeExcludePredicate = + final Optional> optColumnIncludeExcludePredicate = CompiledIncludeExcludeFilter.create( compiledColumn.getColumn().getFilter(), paramMap); - if (optionalColumnIncludeExcludePredicate.isPresent() || needsMapping) { + if (optColumnIncludeExcludePredicate.isPresent() || needsMapping) { final Generator generator = compiledColumn.getGenerator(); if (generator != null) { - final Predicate columnIncludeExcludePredicate = - optionalColumnIncludeExcludePredicate.orElse(Predicates.alwaysTrue()); - final BiConsumer, String> mapConsumer; + final Predicate columnIncludeExcludePredicate = optColumnIncludeExcludePredicate + .orElse(PredicateUtil.ALWAYS_TRUE_STRING_PREDICATE); + + final BiConsumer, String> mapConsumer; if (needsMapping) { mapConsumer = (columnNameToValueMap, s) -> - columnNameToValueMap.put(compiledColumn.getColumn().getName(), s); + columnNameToValueMap.put(compiledColumn.getColumn().getNameAsCIKey(), s); } else { mapConsumer = (columnNameToValueMap, s) -> { }; @@ -71,8 +90,8 @@ public static Predicate create(final ExpressionOperator rowExpression, } // Create a predicate to test the row values. - final Predicate> rowPredicate = optionalRowExpressionMatcher - .map(rowExpressionMatcher -> (Predicate>) columnNameToValueMap -> { + final Predicate> rowPredicate = optRowExpressionMatcher + .map(rowExpressionMatcher -> (Predicate>) columnNameToValueMap -> { try { // Test the row value map. return rowExpressionMatcher.test(columnNameToValueMap); @@ -81,22 +100,19 @@ public static Predicate create(final ExpressionOperator rowExpression, return false; } }) - .orElse(Predicates.alwaysTrue()); + .orElse(RowExpressionMatcher.ALWAYS_TRUE_PREDICATE); // If we need mappings then we need a map to receive them. - final Supplier> mapSupplier; - if (needObjectMap) { - mapSupplier = HashMap::new; - } else { - mapSupplier = Collections::emptyMap; - } + final Supplier> mapSupplier = needObjectMap + ? HashMap::new + : Collections::emptyMap; // If we need column mappings then create a predicate that will use them. if (!usedColumns.isEmpty()) { final ValueReferenceIndex valueReferenceIndex = compiledColumns.getValueReferenceIndex(); return values -> { final StoredValues storedValues = valueReferenceIndex.createStoredValues(); - final Map columnNameToValueMap = mapSupplier.get(); + final Map columnNameToValueMap = mapSupplier.get(); for (final UsedColumn usedColumn : usedColumns) { final Generator generator = usedColumn.generator; @@ -115,19 +131,23 @@ public static Predicate create(final ExpressionOperator rowExpression, // Test the row value map. return rowPredicate.test(columnNameToValueMap); }; - } else if (optionalRowExpressionMatcher.isPresent()) { + } else if (optRowExpressionMatcher.isPresent()) { return values -> { // Test the row value map. return rowPredicate.test(Collections.emptyMap()); }; } else { - return Predicates.alwaysTrue(); + return ALWAYS_TRUE_VAL_PREDICATE; } } + + // -------------------------------------------------------------------------------- + + private record UsedColumn(Generator generator, Predicate columnIncludeExcludePredicate, - BiConsumer, String> mapConsumer) { + BiConsumer, String> mapConsumer) { } } diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/language/DocResolver.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/language/DocResolver.java index 045eb05c59d..6f67e2ec0c5 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/language/DocResolver.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/language/DocResolver.java @@ -4,6 +4,7 @@ import stroom.docref.DocRefInfo; import stroom.docrefinfo.api.DocRefInfoService; import stroom.query.common.v2.DataSourceProviderRegistry; +import stroom.util.NullSafe; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; @@ -50,8 +51,9 @@ public DocRef resolveDocRef(final String type, final String name) { return optionalDocRef.get().getDocRef(); } - // Try by name. - final List docRefs = docRefInfoService.findByName(type, name, false); + // Try by name (case-insensitive). + final List docRefs = docRefInfoService.findByName( + type, name, false, false); if (docRefs.isEmpty()) { throw new RuntimeException(type + " \"" + name + "\" not found"); } else if (docRefs.size() > 1) { @@ -88,7 +90,8 @@ private DocRef getDataSourceRefFromName(final String name) { final List docRefs = dataSourceProviderRegistry .list() .stream() - .filter(docRef -> docRef != null && docRef.getName() != null && docRef.getName().equals(name)) + .filter(docRef -> + NullSafe.equalsIgnoreCase(docRef, DocRef::getName, name)) .toList(); if (docRefs.isEmpty()) { throw new RuntimeException("Data source \"" + diff --git a/stroom-query/stroom-query-common/src/main/java/stroom/query/language/SearchRequestFactory.java b/stroom-query/stroom-query-common/src/main/java/stroom/query/language/SearchRequestFactory.java index 9448f1abea8..2776831ac88 100644 --- a/stroom-query/stroom-query-common/src/main/java/stroom/query/language/SearchRequestFactory.java +++ b/stroom-query/stroom-query-common/src/main/java/stroom/query/language/SearchRequestFactory.java @@ -56,6 +56,7 @@ import stroom.util.NullSafe; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.query.FieldNames; import jakarta.inject.Inject; @@ -66,7 +67,6 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -82,10 +82,16 @@ public class SearchRequestFactory { public static final String TABLE_COMPONENT_ID = "table"; public static final String VIS_COMPONENT_ID = "vis"; + private static final String COLUMN_ID_PREFIX = "column-"; + // Cache cols 0-9 +// private static final CIKey[] COLUMN_ID_CACHE = IntStream.range(0, 10) +// .mapToObj(SearchRequestFactory::createColumnId) +// .map(CIKey::ofStaticKey) +// .toArray(CIKey[]::new); + private final VisualisationTokenConsumer visualisationTokenConsumer; private final DocResolver docResolver; - @Inject public SearchRequestFactory(final VisualisationTokenConsumer visualisationTokenConsumer, final DocResolver docResolver) { @@ -133,7 +139,6 @@ void extractDataSourceOnly(final String string, final Consumer consumer) // Get a list of tokens. final List tokens = Tokeniser.parse(string); - if (tokens.isEmpty()) { throw new TokenException(null, "No tokens"); } @@ -806,19 +811,24 @@ private void addTableSettings(final List tokens, // Ensure StreamId and EventId fields exist if there is no grouping. if (groupDepth == 0) { - if (!addedFields.contains(FieldIndex.FALLBACK_STREAM_ID_FIELD_NAME)) { - tableSettingsBuilder.addColumns(buildSpecialColumn(FieldIndex.FALLBACK_STREAM_ID_FIELD_NAME)); + if (!addedFields.contains(FieldNames.FALLBACK_STREAM_ID_FIELD_KEY.get())) { + tableSettingsBuilder.addColumns(buildSpecialColumn(FieldNames.FALLBACK_STREAM_ID_FIELD_KEY.get())); } - if (!addedFields.contains(FieldIndex.FALLBACK_EVENT_ID_FIELD_NAME)) { - tableSettingsBuilder.addColumns(buildSpecialColumn(FieldIndex.FALLBACK_EVENT_ID_FIELD_NAME)); + if (!addedFields.contains(FieldNames.FALLBACK_EVENT_ID_FIELD_KEY.get())) { + tableSettingsBuilder.addColumns(buildSpecialColumn(FieldNames.FALLBACK_EVENT_ID_FIELD_KEY.get())); } } // Add missing fields if needed. for (final AbstractToken token : additionalFields) { +// final CIKey fieldName = CIKey.of(token.getUnescapedText()); final String fieldName = token.getUnescapedText(); if (!addedFields.contains(fieldName)) { +// final CIKey id = CIKey.of("__" +// + fieldName.get().replaceAll("\\s", "_") +// + "__"); final String id = "__" + fieldName.replaceAll("\\s", "_") + "__"; + tableSettingsBuilder.addColumns(createColumn(token, id, fieldName, @@ -861,6 +871,7 @@ private void addTableSettings(final List tokens, public Column buildSpecialColumn(final String name) { addedFields.add(name); +// final String name = caseInsensitiveName.get(); return Column.builder() .id(name) .name(name) @@ -893,8 +904,8 @@ private void processWindow(final KeywordGroup keywordGroup, final int byIndex = getTokenIndex(children, token -> TokenType.BY.equals(token.getTokenType())); if (byIndex == -1) { throw new TokenException(keywordGroup, "Syntax exception, expected by"); - } else if (children.size() > byIndex + 1) { - final AbstractToken token = children.get(byIndex + 1); + } else if (children.size() > byIndex + 1) { + final AbstractToken token = children.get(byIndex + 1); if (!TokenType.DURATION.equals(token.getTokenType())) { throw new TokenException(token, "Syntax exception, expected valid window duration"); } @@ -903,7 +914,7 @@ private void processWindow(final KeywordGroup keywordGroup, hoppingWindowBuilder.advanceSize(durationString); // We found the duration so remove the tokens. - children.remove(byIndex + 1); + children.remove(byIndex + 1); children.remove(byIndex); } else { throw new TokenException(children.get(byIndex), "Syntax exception, expected window duration"); @@ -1065,7 +1076,8 @@ private void processSelect(final KeywordGroup keywordGroup, } else if (TokenType.COMMA.equals(token.getTokenType())) { if (fieldToken != null) { columnCount++; - final String columnId = "column-" + columnCount; +// final CIKey columnId = getColumnId(columnCount); + final String columnId = createColumnId(columnCount); columns.add(createColumn( fieldToken, columnId, @@ -1079,7 +1091,8 @@ private void processSelect(final KeywordGroup keywordGroup, } else if (fieldExpression != null) { columnCount++; - final String columnId = "column-" + columnCount; +// final CIKey columnId = getColumnId(columnCount); + final String columnId = createColumnId(columnCount); columns.add(createColumn( columnId, fieldExpression, @@ -1102,7 +1115,8 @@ private void processSelect(final KeywordGroup keywordGroup, // Add final field if we have one. if (fieldToken != null) { columnCount++; - final String columnId = "column-" + columnCount; +// final CIKey columnId = getColumnId(columnCount); + final String columnId = createColumnId(columnCount); columns.add(createColumn( fieldToken, columnId, @@ -1116,7 +1130,8 @@ private void processSelect(final KeywordGroup keywordGroup, } else if (fieldExpression != null) { columnCount++; - final String columnId = "column-" + columnCount; +// final CIKey columnId = getColumnId(columnCount); + final String columnId = createColumnId(columnCount); columns.add(createColumn( columnId, fieldExpression, @@ -1218,18 +1233,10 @@ private void processSortBy(final KeywordGroup keywordGroup, if (fieldName == null) { fieldName = t.getUnescapedText(); } else if (direction == null) { - try { - if (t.getUnescapedText().toLowerCase(Locale.ROOT).equalsIgnoreCase("asc")) { - direction = SortDirection.ASCENDING; - } else if (t.getUnescapedText().toLowerCase(Locale.ROOT).equalsIgnoreCase("desc")) { - direction = SortDirection.DESCENDING; - } else { - direction = SortDirection.valueOf(t.getUnescapedText()); - } - } catch (final IllegalArgumentException e) { - throw new TokenException(t, - "Syntax exception, expected sort direction 'asc' or 'desc'"); - } + direction = SortDirection.fromShortForm(t.getUnescapedText()) + .orElseThrow(() -> + new TokenException(t, + "Syntax exception, expected sort direction 'asc' or 'desc'")); } } else if (TokenType.COMMA.equals(t.getTokenType())) { if (fieldName == null) { @@ -1350,4 +1357,18 @@ private static int getTokenIndex(final List tokens, final Predica } return -1; } + + private static String createColumnId(final int colIdx) { + return COLUMN_ID_PREFIX + colIdx; + } + +// private static CIKey getColumnId(final int colIdx) { +// if (colIdx < 0) { +// throw new IllegalArgumentException("Sub-zero column ID index"); +// } else if (colIdx < COLUMN_ID_CACHE.length) { +// return COLUMN_ID_CACHE[colIdx]; +// } else { +// return CIKey.of(createColumnId(colIdx)); +// } +// } } diff --git a/stroom-query/stroom-query-common/src/test/java/stroom/query/common/v2/TestKVMapUtil.java b/stroom-query/stroom-query-common/src/test/java/stroom/query/common/v2/TestKVMapUtil.java index f872cea232d..6d5c60f731a 100644 --- a/stroom-query/stroom-query-common/src/test/java/stroom/query/common/v2/TestKVMapUtil.java +++ b/stroom-query/stroom-query-common/src/test/java/stroom/query/common/v2/TestKVMapUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package stroom.query.common.v2; +import stroom.util.shared.string.CIKey; + import org.junit.jupiter.api.Test; import java.util.Map; @@ -26,9 +28,9 @@ class TestKVMapUtil { @Test void testSimpleParse() { - final Map map = KVMapUtil.parse("param1=value1"); - assertThat(map.keySet().iterator().next()).isEqualTo("param1"); - assertThat(map.get("param1")).isEqualTo("value1"); + final Map map = KVMapUtil.parse("param1=value1"); + assertThat(map.keySet().iterator().next()).isEqualTo(CIKey.of("param1")); + assertThat(map.get(CIKey.of("param1"))).isEqualTo("value1"); } @Test @@ -73,7 +75,7 @@ void testComplexParse() { @Test void testReplacement() { - Map map = KVMapUtil.parse("key1=value1"); + Map map = KVMapUtil.parse("key1=value1"); String result = KVMapUtil.replaceParameters("this is ${key1}", map); assertThat(result).isEqualTo("this is value1"); @@ -99,16 +101,18 @@ void testReplacement() { } private void testKV(String text, String... expectedParams) { - final Map map = KVMapUtil.parse(text); + final Map map = KVMapUtil.parse(text); assertThat(expectedParams.length > 0).isTrue(); assertThat(expectedParams.length % 2 == 0).isTrue(); - assertThat(map).hasSize(expectedParams.length / 2); + assertThat(map) + .hasSize(expectedParams.length / 2); for (int i = 0; i < expectedParams.length; i += 2) { String key = expectedParams[i]; String value = expectedParams[i + 1]; - assertThat(map.get(key)).isEqualTo(value); + assertThat(map.get(CIKey.of(key))) + .isEqualTo(value); } } } diff --git a/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/QueryStoreImpl.java b/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/QueryStoreImpl.java index 6a07c053c32..cdbec93e8e7 100644 --- a/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/QueryStoreImpl.java +++ b/stroom-query/stroom-query-impl/src/main/java/stroom/query/impl/QueryStoreImpl.java @@ -263,8 +263,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/AbstractFunction.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/AbstractFunction.java index 54a9ff4be05..a7540ebb00e 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/AbstractFunction.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/AbstractFunction.java @@ -18,6 +18,7 @@ import stroom.query.language.functions.ref.ValueReferenceIndex; import stroom.query.language.token.Param; +import stroom.util.shared.string.CIKey; import java.text.ParseException; import java.util.Map; @@ -46,11 +47,11 @@ public void setParams(final Param[] params) throws ParseException { } @Override - public void setStaticMappedValues(final Map staticMappedValues) { + public void setStaticMappedValues(final Map staticMappedValues) { if (params != null) { for (final Param param : params) { - if (param instanceof Function) { - ((Function) param).setStaticMappedValues(staticMappedValues); + if (param instanceof final Function function) { + function.setStaticMappedValues(staticMappedValues); } } } @@ -60,8 +61,8 @@ public void setStaticMappedValues(final Map staticMappedValues) public void addValueReferences(final ValueReferenceIndex valueReferenceIndex) { if (params != null) { for (final Param param : params) { - if (param instanceof Function) { - ((Function) param).addValueReferences(valueReferenceIndex); + if (param instanceof final Function function) { + function.addValueReferences(valueReferenceIndex); } } } @@ -98,8 +99,8 @@ void appendParams(final StringBuilder sb) { } void appendParam(final StringBuilder sb, final Param param) { - if (param instanceof Appendable) { - ((Appendable) param).appendString(sb); + if (param instanceof final Appendable appendable) { + appendable.appendString(sb); } else { sb.append(param.toString()); } diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/AbstractStaticFunction.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/AbstractStaticFunction.java index 559af7b11ce..76cdce99d4b 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/AbstractStaticFunction.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/AbstractStaticFunction.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,6 +18,7 @@ import stroom.query.language.functions.ref.ValueReferenceIndex; import stroom.query.language.token.Param; +import stroom.util.shared.string.CIKey; import java.util.Map; @@ -37,7 +38,7 @@ public void setParams(final Param[] params) { } @Override - public void setStaticMappedValues(final Map staticMappedValues) { + public void setStaticMappedValues(final Map staticMappedValues) { // Ignore } diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Concat.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Concat.java index baf087cdcb2..25cc7eee4d6 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Concat.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Concat.java @@ -49,6 +49,10 @@ protected Generator createGenerator(final Generator[] childGenerators) { return new Gen(childGenerators); } + + // -------------------------------------------------------------------------------- + + private static final class Gen extends AbstractManyChildGenerator { Gen(final Generator[] childGenerators) { diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/CurrentUser.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/CurrentUser.java index e2628d78bd6..0770496c282 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/CurrentUser.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/CurrentUser.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,11 +17,12 @@ package stroom.query.language.functions; import stroom.query.language.token.Param; +import stroom.util.shared.GwtNullSafe; +import stroom.util.shared.string.CIKey; import java.text.ParseException; import java.util.HashMap; import java.util.Map; -import java.util.Objects; @SuppressWarnings("unused") //Used by FunctionFactory @FunctionDef( @@ -65,11 +66,11 @@ public void setParams(final Param[] params) throws ParseException { } @Override - public void setStaticMappedValues(final Map staticMappedValues) { + public void setStaticMappedValues(final Map staticMappedValues) { final String v = switch (getNameType()) { - case DISPLAY_NAME -> staticMappedValues.get(ParamKeys.CURRENT_USER); - case SUBJECT_ID -> staticMappedValues.get(ParamKeys.CURRENT_USER_SUBJECT_ID); - case FULL_NAME -> staticMappedValues.get(ParamKeys.CURRENT_USER_FULL_NAME); + case DISPLAY_NAME -> staticMappedValues.get(ParamKeys.CURRENT_USER_KEY); + case SUBJECT_ID -> staticMappedValues.get(ParamKeys.CURRENT_USER_SUBJECT_ID_KEY); + case FULL_NAME -> staticMappedValues.get(ParamKeys.CURRENT_USER_FULL_NAME_KEY); }; if (v != null) { gen = new StaticValueGen(ValString.create(v)); @@ -103,12 +104,12 @@ private enum NameType { SUBJECT_ID("subject"), FULL_NAME("full"), ; - private static final Map STR_TO_ENUM_MAP = new HashMap<>(3); + private static final Map STR_TO_ENUM_MAP = new HashMap<>(3); private static final NameType DEFAULT = DISPLAY_NAME; static { for (final NameType nameType : NameType.values()) { - STR_TO_ENUM_MAP.put(nameType.paramVal, nameType); + STR_TO_ENUM_MAP.put(CIKey.of(nameType.paramVal, ParamKeys.KNOWN_KEYS_MAP), nameType); } } @@ -119,9 +120,10 @@ private enum NameType { } static NameType fromString(final String type) { - return type != null - ? Objects.requireNonNullElse(STR_TO_ENUM_MAP.get(type), DEFAULT) - : DEFAULT; + return GwtNullSafe.getOrElse( + type, + type2 -> STR_TO_ENUM_MAP.get(CIKey.of(type2, ParamKeys.KNOWN_KEYS_MAP)), + DEFAULT); } } } diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Expression.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Expression.java index 2047cb513a3..75cba672ed1 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Expression.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Expression.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import stroom.query.language.functions.ref.ValueReferenceIndex; import stroom.query.language.token.Param; +import stroom.util.shared.string.CIKey; import java.util.Map; @@ -40,7 +41,7 @@ public void setParams(final Param[] params) { } @Override - public void setStaticMappedValues(final Map staticMappedValues) { + public void setStaticMappedValues(final Map staticMappedValues) { if (function != null) { function.setStaticMappedValues(staticMappedValues); } diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/FieldIndex.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/FieldIndex.java index bda108f0f7b..1c0aa6daca9 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/FieldIndex.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/FieldIndex.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,6 +16,10 @@ package stroom.query.language.functions; +import stroom.util.NullSafe; +import stroom.util.shared.query.FieldNames; +import stroom.util.shared.string.CIKey; + import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -27,46 +31,60 @@ public class FieldIndex { - public static final String DEFAULT_TIME_FIELD_NAME = "__time__"; - public static final String FALLBACK_TIME_FIELD_NAME = "EventTime"; - public static final String DEFAULT_STREAM_ID_FIELD_NAME = "__stream_id__"; - public static final String DEFAULT_EVENT_ID_FIELD_NAME = "__event_id__"; - public static final String FALLBACK_STREAM_ID_FIELD_NAME = "StreamId"; - public static final String FALLBACK_EVENT_ID_FIELD_NAME = "EventId"; - - private final Map fieldToPos = new ConcurrentHashMap<>(); + private final Map fieldToPos = new ConcurrentHashMap<>(); - private final List fieldList = new ArrayList<>(); - private volatile String[] posToField = new String[0]; + private final List fieldList = new ArrayList<>(); private Integer timeFieldIndex; private Integer streamIdFieldIndex; private Integer eventIdFieldIndex; public int create(final String fieldName) { - return fieldToPos.computeIfAbsent(fieldName, k -> addField(fieldName)); + final CIKey caseInsensitiveFieldName = CIKey.of(fieldName); + return create(caseInsensitiveFieldName); + } + + public int create(final CIKey caseInsensitiveFieldName) { + final Integer pos = fieldToPos.computeIfAbsent(caseInsensitiveFieldName, k -> + addField(caseInsensitiveFieldName)); + return pos; } - private synchronized int addField(final String fieldName) { + private synchronized int addField(final CIKey fieldName) { fieldList.add(fieldName); - posToField = fieldList.toArray(new String[0]); - return posToField.length - 1; + final int pos = fieldList.size() - 1; + return pos; } public Integer getPos(final String fieldName) { - return fieldToPos.get(fieldName); + return getPos(CIKey.of(fieldName)); + } + + public Integer getPos(final CIKey caseInsensitiveFieldName) { + final Integer pos = fieldToPos.get(caseInsensitiveFieldName); + return pos; } public String getField(final int pos) { - final String[] arr = posToField; - if (pos >= 0 && pos < arr.length) { - return arr[pos]; + return NullSafe.get(getFieldAsCIKey(pos), CIKey::get); + } + + public CIKey getFieldAsCIKey(final int pos) { +// final CIKey[] arr = posToField; + if (pos >= 0 && pos < fieldList.size()) { + return fieldList.get(pos); } return null; } - public String[] getFields() { - return posToField; + public List getFields() { + return fieldList.stream() + .map(CIKey::get) + .toList(); + } + + public List getFieldsAsCIKeys() { + return fieldList; } public int size() { @@ -74,62 +92,56 @@ public int size() { } public Stream> stream() { - return fieldToPos.entrySet().stream(); + return fieldToPos.entrySet() + .stream() + .map(entry -> Map.entry( + NullSafe.get(entry.getKey(), CIKey::get), + entry.getValue())); } public int getTimeFieldIndex() { if (timeFieldIndex == null) { - timeFieldIndex = - Optional.ofNullable(getPos(DEFAULT_TIME_FIELD_NAME)) - .or(() -> Optional.ofNullable(getPos(FALLBACK_TIME_FIELD_NAME))) + final int idx = + Optional.ofNullable(getPos(FieldNames.DEFAULT_TIME_FIELD_KEY)) + .or(() -> Optional.ofNullable(getPos(FieldNames.FALLBACK_TIME_FIELD_KEY))) .orElse(-1); + timeFieldIndex = idx; + return idx; } return timeFieldIndex; } - /** - * @return True if fieldName matches the special Stream ID field. - */ - public static boolean isStreamIdFieldName(final String fieldName) { - return Objects.equals(DEFAULT_STREAM_ID_FIELD_NAME, fieldName) - || Objects.equals(FALLBACK_STREAM_ID_FIELD_NAME, fieldName); - } - public int getStreamIdFieldIndex() { if (streamIdFieldIndex == null) { - streamIdFieldIndex = - Optional.ofNullable(getPos(DEFAULT_STREAM_ID_FIELD_NAME)) + final int idx = + Optional.ofNullable(getPos(FieldNames.DEFAULT_STREAM_ID_FIELD_KEY)) .or(() -> Optional.ofNullable( - getPos(FALLBACK_STREAM_ID_FIELD_NAME))) + getPos(FieldNames.FALLBACK_STREAM_ID_FIELD_KEY))) .or(() -> { - create(FALLBACK_STREAM_ID_FIELD_NAME); + create(FieldNames.FALLBACK_STREAM_ID_FIELD_KEY); return Optional.ofNullable( - getPos(FALLBACK_STREAM_ID_FIELD_NAME)); + getPos(FieldNames.FALLBACK_STREAM_ID_FIELD_KEY)); }) .orElse(-1); + this.streamIdFieldIndex = idx; + return idx; } return streamIdFieldIndex; } - /** - * @return True if fieldName matches the special Event ID field. - */ - public static boolean isEventIdFieldName(final String fieldName) { - return Objects.equals(DEFAULT_EVENT_ID_FIELD_NAME, fieldName) - || Objects.equals(FALLBACK_EVENT_ID_FIELD_NAME, fieldName); - } - public int getEventIdFieldIndex() { if (eventIdFieldIndex == null) { - eventIdFieldIndex = - Optional.ofNullable(getPos(DEFAULT_EVENT_ID_FIELD_NAME)) + final int idx = + Optional.ofNullable(getPos(FieldNames.DEFAULT_EVENT_ID_FIELD_KEY)) .or(() -> Optional.ofNullable( - getPos(FALLBACK_EVENT_ID_FIELD_NAME))) + getPos(FieldNames.FALLBACK_EVENT_ID_FIELD_KEY))) .or(() -> { - create(FALLBACK_EVENT_ID_FIELD_NAME); + create(FieldNames.FALLBACK_EVENT_ID_FIELD_KEY); return Optional.ofNullable( - getPos(FALLBACK_EVENT_ID_FIELD_NAME)); + getPos(FieldNames.FALLBACK_EVENT_ID_FIELD_KEY)); }).orElse(-1); + eventIdFieldIndex = idx; + return idx; } return eventIdFieldIndex; } diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Function.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Function.java index cde8d2d3dea..46d917e61e3 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Function.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/Function.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,6 +18,7 @@ import stroom.query.language.functions.ref.ValueReferenceIndex; import stroom.query.language.token.Param; +import stroom.util.shared.string.CIKey; import java.text.ParseException; import java.util.Map; @@ -38,7 +39,7 @@ public interface Function extends Param { * * @param staticMappedValues The static mapped values for the Param and Params functions to use. */ - void setStaticMappedValues(Map staticMappedValues); + void setStaticMappedValues(Map staticMappedValues); void addValueReferences(ValueReferenceIndex valueReferenceIndex); diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/ParamFactory.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/ParamFactory.java index 4c20cd58eb3..4126665c96d 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/ParamFactory.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/ParamFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,31 +34,21 @@ public Param create(final FieldIndex fieldIndex, final Token token) { try { // Token should be string or number or field. - switch (token.getTokenType()) { - case DOUBLE_QUOTED_STRING, SINGLE_QUOTED_STRING: - return ValString.create(token.getUnescapedText()); - - case STRING, PARAM: - return createRef(token.getUnescapedText(), fieldIndex); - - case DATE_TIME: - return ValDate.create(DateUtil.parseNormalDateTimeString(token.getText())); - - case DURATION: - return ValDuration.create(ValDurationUtil.parseToMilliseconds(token.getText())); - - case NUMBER: - return ValDouble.create(Double.parseDouble(token.getText())); - - default: - throw new TokenException(token, "Unexpected token type '" + token.getTokenType() + "'"); - } + return switch (token.getTokenType()) { + case DOUBLE_QUOTED_STRING, SINGLE_QUOTED_STRING -> ValString.create(token.getUnescapedText()); + case STRING, PARAM -> createRef(token.getUnescapedText(), fieldIndex); + case DATE_TIME -> ValDate.create(DateUtil.parseNormalDateTimeString(token.getText())); + case DURATION -> ValDuration.create(ValDurationUtil.parseToMilliseconds(token.getText())); + case NUMBER -> ValDouble.create(Double.parseDouble(token.getText())); + default -> throw new TokenException(token, "Unexpected token type '" + token.getTokenType() + "'"); + }; } catch (final RuntimeException e) { throw new TokenException(token, e.getMessage()); } } private Param createRef(final String name, final FieldIndex fieldIndex) { +// final CIKey caseInsensitiveFieldName = CIKey.of(name); final Expression expression = expressionReference.get(name); if (expression != null) { return expression; diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/ParamKeys.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/ParamKeys.java index 912a82c1d1c..c751b575252 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/ParamKeys.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/ParamKeys.java @@ -1,5 +1,24 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.language.functions; +import stroom.util.shared.string.CIKey; + +import java.util.Map; import java.util.Set; public class ParamKeys { @@ -8,23 +27,39 @@ public class ParamKeys { * The display name (or subjectId if there isn't one) of the current logged-in user */ public static final String CURRENT_USER = "currentUser()"; + public static final CIKey CURRENT_USER_KEY = CIKey.ofStaticKey(CURRENT_USER); /** - * The subjectId of the curent logged-in user + * The subjectId of the current logged-in user */ public static final String CURRENT_USER_SUBJECT_ID = "currentUserSubjectId()"; + public static final CIKey CURRENT_USER_SUBJECT_ID_KEY = CIKey.ofStaticKey(CURRENT_USER_SUBJECT_ID); /** - * The full name of the curent logged-in user. May be null + * The full name of the current logged-in user. May be null */ public static final String CURRENT_USER_FULL_NAME = "currentUserFullName()"; + public static final CIKey CURRENT_USER_FULL_NAME_KEY = CIKey.ofStaticKey(CURRENT_USER_FULL_NAME); - static final Set INTERNAL_PARAM_KEYS = Set.of( - CURRENT_USER, - CURRENT_USER_SUBJECT_ID, - CURRENT_USER_FULL_NAME + private static final Set INTERNAL_PARAM_KEYS_AS_KEYS = Set.of( + CURRENT_USER_KEY, + CURRENT_USER_SUBJECT_ID_KEY, + CURRENT_USER_FULL_NAME_KEY ); + public static final Map KNOWN_KEYS_MAP = Map.of( + CURRENT_USER, CURRENT_USER_KEY, + CURRENT_USER_SUBJECT_ID, CURRENT_USER_SUBJECT_ID_KEY, + CURRENT_USER_FULL_NAME, CURRENT_USER_FULL_NAME_KEY); + + static boolean isInternalParamKey(final CIKey key) { + if (key == null) { + return false; + } else { + return INTERNAL_PARAM_KEYS_AS_KEYS.contains(key); + } + } + private ParamKeys() { } } diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/QueryParam.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/QueryParam.java index f982ef74d81..fb9f8f0125b 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/QueryParam.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/QueryParam.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -17,6 +17,7 @@ package stroom.query.language.functions; import stroom.query.language.token.Param; +import stroom.util.shared.string.CIKey; import java.text.ParseException; import java.util.Map; @@ -66,12 +67,12 @@ public void setParams(final Param[] params) throws ParseException { } @Override - public void setStaticMappedValues(final Map staticMappedValues) { + public void setStaticMappedValues(final Map staticMappedValues) { if (key == null) { throw new RuntimeException("Key must be set before calling setStaticMappedValues"); } - final String v = staticMappedValues.get(key); + final String v = staticMappedValues.get(CIKey.of(key)); if (v != null) { gen = new StaticValueGen(ValString.create(v)); } diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/QueryParams.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/QueryParams.java index c70b8070077..447a4752c9b 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/QueryParams.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/QueryParams.java @@ -1,11 +1,11 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,6 +16,8 @@ package stroom.query.language.functions; +import stroom.util.shared.string.CIKey; + import java.util.Map; import java.util.stream.Collectors; @@ -39,11 +41,12 @@ public QueryParams(final String name) { } @Override - public void setStaticMappedValues(final Map staticMappedValues) { + public void setStaticMappedValues(final Map staticMappedValues) { if (staticMappedValues != null) { final String str = staticMappedValues.entrySet() .stream() - .filter(entry -> !ParamKeys.INTERNAL_PARAM_KEYS.contains(entry.getKey())) + .filter(entry -> + !ParamKeys.isInternalParamKey(entry.getKey())) .map(entry -> entry.getKey() + "=\"" + entry.getValue() + "\"") .collect(Collectors.joining(" ")); diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/StaticValueFunction.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/StaticValueFunction.java index a94b170b50c..d152820f396 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/StaticValueFunction.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/functions/StaticValueFunction.java @@ -18,6 +18,7 @@ import stroom.query.language.functions.ref.ValueReferenceIndex; import stroom.query.language.token.Param; +import stroom.util.shared.string.CIKey; import java.util.Map; @@ -46,7 +47,7 @@ public void setParams(final Param[] params) { } @Override - public void setStaticMappedValues(final Map staticMappedValues) { + public void setStaticMappedValues(final Map staticMappedValues) { // Ignore } diff --git a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/token/FunctionGroup.java b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/token/FunctionGroup.java index 691e8816286..a3897220a2f 100644 --- a/stroom-query/stroom-query-language/src/main/java/stroom/query/language/token/FunctionGroup.java +++ b/stroom-query/stroom-query-language/src/main/java/stroom/query/language/token/FunctionGroup.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.query.language.token; import java.util.List; @@ -28,6 +44,10 @@ void appendAttributes(final StringBuilder sb) { sb.append("\""); } + + // -------------------------------------------------------------------------------- + + public static class Builder extends AbstractTokenGroupBuilder { private String name; diff --git a/stroom-query/stroom-query-language/src/test/java/stroom/query/language/functions/AbstractExpressionParserTest.java b/stroom-query/stroom-query-language/src/test/java/stroom/query/language/functions/AbstractExpressionParserTest.java index b09e1e734b4..8d793e9ef56 100644 --- a/stroom-query/stroom-query-language/src/test/java/stroom/query/language/functions/AbstractExpressionParserTest.java +++ b/stroom-query/stroom-query-language/src/test/java/stroom/query/language/functions/AbstractExpressionParserTest.java @@ -20,6 +20,7 @@ import stroom.query.language.functions.ref.KryoDataWriter; import stroom.query.language.functions.ref.StoredValues; import stroom.query.language.functions.ref.ValueReferenceIndex; +import stroom.util.shared.string.CIKey; import com.esotericsoftware.kryo.io.ByteBufferInput; import com.esotericsoftware.kryo.unsafe.UnsafeByteBufferOutput; @@ -266,8 +267,8 @@ protected void createExpression(final String expression, throw new RuntimeException(e.getMessage(), e); } - final Map mappedValues = new HashMap<>(); - mappedValues.put("testkey", "testvalue"); + final Map mappedValues = new HashMap<>(); + mappedValues.put(CIKey.of("testkey"), "testvalue"); exp.setStaticMappedValues(mappedValues); final String actual = exp.toString(); diff --git a/stroom-query/stroom-query-language/src/test/java/stroom/query/language/functions/TestFieldIndex.java b/stroom-query/stroom-query-language/src/test/java/stroom/query/language/functions/TestFieldIndex.java new file mode 100644 index 00000000000..32c166c07e1 --- /dev/null +++ b/stroom-query/stroom-query-language/src/test/java/stroom/query/language/functions/TestFieldIndex.java @@ -0,0 +1,164 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.query.language.functions; + +import stroom.util.shared.query.FieldNames; +import stroom.util.shared.string.CIKey; + +import org.junit.jupiter.api.Test; + +import java.util.Map.Entry; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static stroom.util.shared.query.FieldNames.DEFAULT_STREAM_ID_FIELD_NAME; +import static stroom.util.shared.query.FieldNames.DEFAULT_TIME_FIELD_NAME; + +class TestFieldIndex { + + @Test + void test() { + final FieldIndex fieldIndex = new FieldIndex(); + fieldIndex.create("foo"); + fieldIndex.create("FOO"); // same case-insensitive key, so ignored + fieldIndex.create("BAR"); + + assertThat(fieldIndex.getFields()) + .containsExactly("foo", "BAR"); + + assertThat(fieldIndex.stream().map(Entry::getKey).collect(Collectors.toSet())) + .containsExactlyInAnyOrder("foo", "BAR"); + assertThat(fieldIndex.getFieldsAsCIKeys()) + .containsExactly(CIKey.of("foo"), CIKey.of("BAR")); + + assertThat(fieldIndex.getPos("xxx")) + .isEqualTo(null); + assertThat(fieldIndex.getPos("foo")) + .isEqualTo(0); + assertThat(fieldIndex.getPos("FOO")) + .isEqualTo(0); + assertThat(fieldIndex.getPos("bar")) + .isEqualTo(1); + + assertThat(fieldIndex.size()) + .isEqualTo(2); + + assertThat(fieldIndex.getField(0)) + .isEqualTo("foo"); + assertThat(fieldIndex.getField(1)) + .isEqualTo("BAR"); + } + + @Test + void test2() { + final FieldIndex fieldIndex = new FieldIndex(); + fieldIndex.create(CIKey.of("foo")); + fieldIndex.create(CIKey.of("FOO")); // same case-insensitive key, so ignored + fieldIndex.create(CIKey.of("BAR")); + + assertThat(fieldIndex.getFields()) + .containsExactly("foo", "BAR"); + + assertThat(fieldIndex.getFieldsAsCIKeys()) + .containsExactly(CIKey.of("foo"), CIKey.of("BAR")); + + assertThat(fieldIndex.stream().map(Entry::getKey).collect(Collectors.toSet())) + .containsExactlyInAnyOrder("foo", "BAR"); + + assertThat(fieldIndex.getPos("xxx")) + .isEqualTo(null); + assertThat(fieldIndex.getPos("foo")) + .isEqualTo(0); + assertThat(fieldIndex.getPos("FOO")) + .isEqualTo(0); + assertThat(fieldIndex.getPos("bar")) + .isEqualTo(1); + + assertThat(fieldIndex.size()) + .isEqualTo(2); + + assertThat(fieldIndex.getField(0)) + .isEqualTo("foo"); + assertThat(fieldIndex.getField(1)) + .isEqualTo("BAR"); + } + + @Test + void getStreamIdFieldIndex() { + final FieldIndex fieldIndex = new FieldIndex(); + assertThat(fieldIndex.getStreamIdFieldIndex()) + .isEqualTo(fieldIndex.getPos(FieldNames.FALLBACK_STREAM_ID_FIELD_NAME)); + + assertThat(fieldIndex.getStreamIdFieldIndex()) + .isEqualTo(fieldIndex.getPos(FieldNames.FALLBACK_STREAM_ID_FIELD_NAME)); + } + + @Test + void getStreamIdFieldIndex2() { + final FieldIndex fieldIndex = new FieldIndex(); + fieldIndex.create(DEFAULT_STREAM_ID_FIELD_NAME); + + assertThat(fieldIndex.getStreamIdFieldIndex()) + .isEqualTo(fieldIndex.getPos(DEFAULT_STREAM_ID_FIELD_NAME)); + + assertThat(fieldIndex.getStreamIdFieldIndex()) + .isEqualTo(fieldIndex.getPos(DEFAULT_STREAM_ID_FIELD_NAME)); + } + + @Test + void getEventIdFieldIndex() { + final FieldIndex fieldIndex = new FieldIndex(); + assertThat(fieldIndex.getEventIdFieldIndex()) + .isEqualTo(fieldIndex.getPos(FieldNames.FALLBACK_EVENT_ID_FIELD_NAME)); + + assertThat(fieldIndex.getEventIdFieldIndex()) + .isEqualTo(fieldIndex.getPos(FieldNames.FALLBACK_EVENT_ID_FIELD_NAME)); + } + + @Test + void getEventIdFieldIndex2() { + final FieldIndex fieldIndex = new FieldIndex(); + fieldIndex.create(FieldNames.DEFAULT_EVENT_ID_FIELD_NAME); + + assertThat(fieldIndex.getEventIdFieldIndex()) + .isEqualTo(fieldIndex.getPos(FieldNames.DEFAULT_EVENT_ID_FIELD_NAME)); + + assertThat(fieldIndex.getEventIdFieldIndex()) + .isEqualTo(fieldIndex.getPos(FieldNames.DEFAULT_EVENT_ID_FIELD_NAME)); + } + + @Test + void getTimeFieldIndex() { + final FieldIndex fieldIndex = new FieldIndex(); + assertThat(fieldIndex.getTimeFieldIndex()) + .isEqualTo(-1); + assertThat(fieldIndex.getTimeFieldIndex()) + .isEqualTo(-1); + } + + @Test + void getTimeFieldIndex2() { + final FieldIndex fieldIndex = new FieldIndex(); + fieldIndex.create(DEFAULT_TIME_FIELD_NAME); + + assertThat(fieldIndex.getTimeFieldIndex()) + .isEqualTo(fieldIndex.getPos(DEFAULT_TIME_FIELD_NAME)); + + assertThat(fieldIndex.getTimeFieldIndex()) + .isEqualTo(fieldIndex.getPos(DEFAULT_TIME_FIELD_NAME)); + } +} diff --git a/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/RequestAuthenticator.java b/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/RequestAuthenticator.java index 2be72c2bcf6..2938f13d22d 100644 --- a/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/RequestAuthenticator.java +++ b/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/RequestAuthenticator.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.receive.common; import stroom.meta.api.AttributeMap; @@ -14,6 +30,7 @@ public interface RequestAuthenticator { /** * Authenticate an inbound request + * * @return */ UserIdentity authenticate(final HttpServletRequest request, @@ -22,15 +39,16 @@ UserIdentity authenticate(final HttpServletRequest request, /** * Check for presence of tokens/certs on an inbound request that determines if authentication * is possible. + * * @return True if the request has the required heaader(s) for authentication. */ boolean hasAuthenticationToken(final HttpServletRequest request); - /** - * Remove any headers relating to authorisations, e.g. 'Authorisation', - * from the passed map - */ - void removeAuthorisationEntries(final Map headers); +// /** +// * Remove any headers relating to authorisations, e.g. 'Authorisation', +// * from the passed map +// */ +// void removeAuthorisationEntries(final Map headers); /** * @return The authentication/authorisation headers to enable authentication with this user diff --git a/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/RequestAuthenticatorImpl.java b/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/RequestAuthenticatorImpl.java index 079b369da07..67131beb97a 100644 --- a/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/RequestAuthenticatorImpl.java +++ b/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/RequestAuthenticatorImpl.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.receive.common; import stroom.meta.api.AttributeMap; @@ -9,6 +25,7 @@ import stroom.util.cert.CertificateExtractor; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -151,8 +168,7 @@ public boolean hasAuthenticationToken(final HttpServletRequest request) { return userIdentityFactory.hasAuthenticationToken(request); } - @Override - public void removeAuthorisationEntries(final Map headers) { + private void removeAuthorisationEntries(final Map headers) { NullSafe.consume(headers, userIdentityFactory::removeAuthEntries); } diff --git a/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/StreamFactory.java b/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/StreamFactory.java index 242f1c757de..eba59a61ced 100644 --- a/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/StreamFactory.java +++ b/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/StreamFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,10 @@ package stroom.receive.common; +import stroom.meta.api.AttributeMap; import stroom.meta.api.StandardHeaderArguments; import stroom.util.date.DateUtil; - -import java.util.Map; +import stroom.util.shared.string.CIKey; /** * Helper to build meta data classes. @@ -27,19 +27,20 @@ public final class StreamFactory { @Deprecated - private static final String PERIOD_START_TIME = "periodStartTime"; + private static final CIKey PERIOD_START_TIME = CIKey.ofStaticKey("periodStartTime"); private StreamFactory() { // Private constructor. } - public static Long getReferenceEffectiveTime(final Map argsMap, boolean doDefault) { - Long effectiveMs = getSafeMs(argsMap, StandardHeaderArguments.EFFECTIVE_TIME); + public static Long getReferenceEffectiveTime(final AttributeMap attributeMap, + boolean doDefault) { + Long effectiveMs = getSafeMs(attributeMap, StandardHeaderArguments.EFFECTIVE_TIME); if (effectiveMs != null) { return effectiveMs; } // This is here for backwards compatibility - effectiveMs = getSafeMs(argsMap, PERIOD_START_TIME); + effectiveMs = getSafeMs(attributeMap, PERIOD_START_TIME); if (effectiveMs != null) { return effectiveMs; } @@ -49,8 +50,8 @@ public static Long getReferenceEffectiveTime(final Map argsMap, return null; } - public static Long getReceivedTime(final Map argsMap, boolean doDefault) { - Long receivedTimeMs = getSafeMs(argsMap, StandardHeaderArguments.RECEIVED_TIME); + public static Long getReceivedTime(final AttributeMap attributeMap, boolean doDefault) { + Long receivedTimeMs = getSafeMs(attributeMap, StandardHeaderArguments.RECEIVED_TIME); if (receivedTimeMs != null) { return receivedTimeMs; } @@ -63,8 +64,8 @@ public static Long getReceivedTime(final Map argsMap, boolean do /** * Helper to avoid null pointers */ - private static Long getSafeMs(final Map argsMap, final String key) { - String value = argsMap.get(key); + private static Long getSafeMs(final AttributeMap attributeMap, final CIKey key) { + String value = attributeMap.get(key); if (value != null) { try { return DateUtil.parseNormalDateTimeString(value); diff --git a/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/StreamTargetStreamHandler.java b/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/StreamTargetStreamHandler.java index 7549717412a..772750fe8ab 100644 --- a/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/StreamTargetStreamHandler.java +++ b/stroom-receive/stroom-receive-common/src/main/java/stroom/receive/common/StreamTargetStreamHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.receive.common; @@ -161,7 +160,7 @@ public long addEntry(final String entryName, putAll(globalAttributeMap, metaAttributeMap); if (metaDataStatistics != null) { - metaDataStatistics.recordStatistics(metaAttributeMap); + metaDataStatistics.recordStatistics(metaAttributeMap.asUnmodifiableMap()); } // Are we switching feed? diff --git a/stroom-receive/stroom-receive-common/src/test/java/stroom/receive/common/TestRequestAuthenticatorImpl.java b/stroom-receive/stroom-receive-common/src/test/java/stroom/receive/common/TestRequestAuthenticatorImpl.java index 16fde3816bb..7dfe4b82003 100644 --- a/stroom-receive/stroom-receive-common/src/test/java/stroom/receive/common/TestRequestAuthenticatorImpl.java +++ b/stroom-receive/stroom-receive-common/src/test/java/stroom/receive/common/TestRequestAuthenticatorImpl.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.receive.common; import stroom.meta.api.AttributeMap; @@ -7,6 +23,7 @@ import stroom.security.api.UserIdentityFactory; import stroom.test.common.TestUtil; import stroom.util.cert.CertificateExtractor; +import stroom.util.shared.string.CIKeys; import io.vavr.Tuple; import jakarta.servlet.http.HttpServletRequest; @@ -104,7 +121,7 @@ Stream authenticate_tokenEnabled() { Assertions.assertThat(attributeMap.size()) .isEqualTo(2); // Two new items added Assertions.assertThat(attributeMap.keySet()) - .doesNotContain(HttpHeaders.AUTHORIZATION); + .doesNotContain(CIKeys.AUTHORIZATION); Assertions.assertThat(attributeMap.get(StandardHeaderArguments.UPLOAD_USER_ID)) .isEqualTo(testOutcome.getActualOutput()._1.getSubjectId()); } else { diff --git a/stroom-receive/stroom-receive-common/src/test/java/stroom/receive/common/TestStroomStreamProcessor.java b/stroom-receive/stroom-receive-common/src/test/java/stroom/receive/common/TestStroomStreamProcessor.java index 67c6f4f9530..c14b9acae3d 100644 --- a/stroom-receive/stroom-receive-common/src/test/java/stroom/receive/common/TestStroomStreamProcessor.java +++ b/stroom-receive/stroom-receive-common/src/test/java/stroom/receive/common/TestStroomStreamProcessor.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.receive.common; @@ -448,7 +464,7 @@ void testProcessRequestHeader_populatedMap() { .isEqualTo(guid); Assertions.assertThat(attributeMap.get(StandardHeaderArguments.RECEIVED_TIME)) .isEqualTo(DateUtil.createNormalDateTimeString(now)); - Assertions.assertThat(attributeMap.getAsList(StandardHeaderArguments.RECEIVED_TIME_HISTORY)) + Assertions.assertThat(attributeMap.getValueAsList(StandardHeaderArguments.RECEIVED_TIME_HISTORY)) .containsExactly( DateUtil.createNormalDateTimeString(prevTime), DateUtil.createNormalDateTimeString(now)); @@ -489,7 +505,7 @@ void testProcessRequestHeader_populatedMap2() { .isEqualTo(guid); Assertions.assertThat(attributeMap.get(StandardHeaderArguments.RECEIVED_TIME)) .isEqualTo(DateUtil.createNormalDateTimeString(now)); - Assertions.assertThat(attributeMap.getAsList(StandardHeaderArguments.RECEIVED_TIME_HISTORY)) + Assertions.assertThat(attributeMap.getValueAsList(StandardHeaderArguments.RECEIVED_TIME_HISTORY)) .containsExactly( DateUtil.createNormalDateTimeString(prevTime), DateUtil.createNormalDateTimeString(now)); diff --git a/stroom-receive/stroom-receive-rules-impl/src/main/java/stroom/receive/rules/impl/ReceiveDataPolicyChecker.java b/stroom-receive/stroom-receive-rules-impl/src/main/java/stroom/receive/rules/impl/ReceiveDataPolicyChecker.java index 28c9c581b73..b080d88c81d 100644 --- a/stroom-receive/stroom-receive-rules-impl/src/main/java/stroom/receive/rules/impl/ReceiveDataPolicyChecker.java +++ b/stroom-receive/stroom-receive-rules-impl/src/main/java/stroom/receive/rules/impl/ReceiveDataPolicyChecker.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.receive.rules.impl; @@ -22,10 +21,13 @@ import stroom.expression.matcher.ExpressionMatcher; import stroom.expression.matcher.ExpressionMatcherFactory; import stroom.meta.api.AttributeMap; +import stroom.query.api.v2.ExpressionOperator; import stroom.query.api.v2.ExpressionUtil; import stroom.receive.rules.shared.ReceiveDataRule; import stroom.receive.rules.shared.ReceiveDataRules; import stroom.receive.rules.shared.RuleAction; +import stroom.util.NullSafe; +import stroom.util.shared.string.CIKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,18 +93,21 @@ private synchronized void refresh() { && dataReceiptPolicy.getRules() != null && dataReceiptPolicy.getFields() != null) { // Create a map of fields. - final Map fieldMap = dataReceiptPolicy.getFields() + final Map fieldMap = dataReceiptPolicy.getFields() .stream() - .collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + .collect(Collectors.toMap( + (QueryField queryField1) -> CIKey.of(queryField1.getFldName()), + Function.identity())); // Also make sure we create a list of rules that are enabled and have at least one enabled term. final Set fieldSet = new HashSet<>(); final List activeRules = new ArrayList<>(); dataReceiptPolicy.getRules().forEach(rule -> { - if (rule.isEnabled() && rule.getExpression() != null && rule.getExpression().enabled()) { + if (rule.isEnabled() + && NullSafe.test(rule.getExpression(), ExpressionOperator::enabled)) { final Set set = new HashSet<>(); addToFieldSet(rule, set); - if (set.size() > 0) { + if (!set.isEmpty()) { fieldSet.addAll(set); } // expression may have no fields in it. @@ -111,11 +116,13 @@ private synchronized void refresh() { }); // Create a map of fields that are valid fields and have been used in the expressions. - final Map usedFieldMap = fieldSet + final Map usedFieldMap = fieldSet .stream() .map(fieldMap::get) .filter(Objects::nonNull) - .collect(Collectors.toMap(QueryField::getFldName, Function.identity())); + .collect(Collectors.toMap( + (QueryField queryField) -> CIKey.of(queryField.getFldName()), + Function.identity())); final ExpressionMatcher expressionMatcher = expressionMatcherFactory.create(usedFieldMap); checker = new CheckerImpl(expressionMatcher, activeRules, fieldMap); @@ -134,11 +141,19 @@ private void addToFieldSet(final ReceiveDataRule rule, final Set fieldSe } } + + // -------------------------------------------------------------------------------- + + private interface Checker { RuleAction check(AttributeMap attributeMap); } + + // -------------------------------------------------------------------------------- + + private static class ReceiveAllChecker implements Checker { @Override @@ -147,15 +162,19 @@ public RuleAction check(final AttributeMap attributeMap) { } } + + // -------------------------------------------------------------------------------- + + private static class CheckerImpl implements Checker { private final ExpressionMatcher expressionMatcher; private final List activeRules; - private final Map fieldMap; + private final Map fieldMap; CheckerImpl(final ExpressionMatcher expressionMatcher, final List activeRules, - final Map fieldMap) { + final Map fieldMap) { this.expressionMatcher = expressionMatcher; this.activeRules = activeRules; this.fieldMap = fieldMap; @@ -163,7 +182,7 @@ private static class CheckerImpl implements Checker { @Override public RuleAction check(final AttributeMap attributeMap) { - final Map map = createAttributeMap(attributeMap, fieldMap); + final Map map = createAttributeMap(attributeMap, fieldMap); final ReceiveDataRule matchingRule = findMatchingRule(expressionMatcher, map, activeRules); if (matchingRule != null && matchingRule.getAction() != null) { @@ -174,12 +193,12 @@ public RuleAction check(final AttributeMap attributeMap) { return RuleAction.RECEIVE; } - private Map createAttributeMap(final AttributeMap attributeMap, - final Map fieldMap) { - final Map map = new HashMap<>(); + private Map createAttributeMap(final AttributeMap attributeMap, + final Map fieldMap) { + final Map map = new HashMap<>(); fieldMap.forEach((fieldName, field) -> { try { - final String string = attributeMap.get(fieldName); + final String string = attributeMap.get(fieldName.get()); switch (field.getFldType()) { case TEXT: map.put(fieldName, string); @@ -208,7 +227,7 @@ private Long getSafeLong(final String string) { } private ReceiveDataRule findMatchingRule(final ExpressionMatcher expressionMatcher, - final Map attributeMap, + final Map attributeMap, final List activeRules) { for (final ReceiveDataRule rule : activeRules) { try { diff --git a/stroom-receive/stroom-receive-rules-impl/src/main/java/stroom/receive/rules/impl/ReceiveDataRuleSetServiceImpl.java b/stroom-receive/stroom-receive-rules-impl/src/main/java/stroom/receive/rules/impl/ReceiveDataRuleSetServiceImpl.java index 517f2ef3c76..c5593434a6b 100644 --- a/stroom-receive/stroom-receive-rules-impl/src/main/java/stroom/receive/rules/impl/ReceiveDataRuleSetServiceImpl.java +++ b/stroom-receive/stroom-receive-rules-impl/src/main/java/stroom/receive/rules/impl/ReceiveDataRuleSetServiceImpl.java @@ -209,8 +209,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-search/stroom-expression-matcher/build.gradle b/stroom-search/stroom-expression-matcher/build.gradle index e94ddc0f514..70583a40fb1 100644 --- a/stroom-search/stroom-expression-matcher/build.gradle +++ b/stroom-search/stroom-expression-matcher/build.gradle @@ -7,13 +7,14 @@ dependencies { implementation project(':stroom-query:stroom-query-language-api') implementation project(':stroom-query:stroom-query-api') implementation project(':stroom-query:stroom-query-common') + implementation project(':stroom-util-shared') + implementation project(':stroom-util') implementation libs.jackson_annotations implementation libs.jakarta_inject implementation libs.jaxb_api testImplementation project(':stroom-core-shared') - testImplementation project(':stroom-util-shared') testImplementation libs.assertj_core testImplementation libs.junit_jupiter_api diff --git a/stroom-search/stroom-expression-matcher/src/main/java/stroom/expression/matcher/ExpressionMatcher.java b/stroom-search/stroom-expression-matcher/src/main/java/stroom/expression/matcher/ExpressionMatcher.java index f32059dc57c..b4405a13228 100644 --- a/stroom-search/stroom-expression-matcher/src/main/java/stroom/expression/matcher/ExpressionMatcher.java +++ b/stroom-search/stroom-expression-matcher/src/main/java/stroom/expression/matcher/ExpressionMatcher.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.expression.matcher; @@ -28,6 +27,8 @@ import stroom.query.api.v2.ExpressionTerm; import stroom.query.api.v2.ExpressionTerm.Condition; import stroom.query.common.v2.DateExpressionParser; +import stroom.util.NullSafe; +import stroom.util.shared.string.CIKey; import java.util.Collection; import java.util.List; @@ -41,31 +42,31 @@ public class ExpressionMatcher { private static final String DELIMITER = ","; - private final Map fieldMap; + private final Map fieldNameToFieldMap; private final WordListProvider wordListProvider; private final CollectionService collectionService; private final Map wordMap = new ConcurrentHashMap<>(); - private final Map patternMap = new ConcurrentHashMap<>(); + private final Map termValueToPatternMap = new ConcurrentHashMap<>(); private final DateTimeSettings dateTimeSettings; - public ExpressionMatcher(final Map fieldMap) { - this.fieldMap = fieldMap; + public ExpressionMatcher(final Map fieldNameToFieldMap) { + this.fieldNameToFieldMap = fieldNameToFieldMap; this.wordListProvider = null; this.collectionService = null; this.dateTimeSettings = DateTimeSettings.builder().build(); } - public ExpressionMatcher(final Map fieldMap, + public ExpressionMatcher(final Map fieldNameToFieldMap, final WordListProvider wordListProvider, final CollectionService collectionService, final DateTimeSettings dateTimeSettings) { - this.fieldMap = fieldMap; + this.fieldNameToFieldMap = fieldNameToFieldMap; this.wordListProvider = wordListProvider; this.collectionService = collectionService; this.dateTimeSettings = dateTimeSettings; } - public boolean match(final Map attributeMap, final ExpressionItem item) { + public boolean match(final Map attributeMap, final ExpressionItem item) { // If the initial item is null or not enabled then don't match. if (item == null || !item.enabled()) { return true; @@ -73,7 +74,7 @@ public boolean match(final Map attributeMap, final ExpressionIte return matchItem(attributeMap, item); } - private boolean matchItem(final Map attributeMap, final ExpressionItem item) { + private boolean matchItem(final Map attributeMap, final ExpressionItem item) { if (!item.enabled()) { // If the child item is not enabled then return and keep trying to match with other parts of the expression. return true; @@ -88,56 +89,50 @@ private boolean matchItem(final Map attributeMap, final Expressi } } - private boolean matchOperator(final Map attributeMap, + private boolean matchOperator(final Map attributeMap, final ExpressionOperator operator) { if (!operator.hasEnabledChildren()) { return true; } else { final List enabledChildren = operator.getEnabledChildren(); - switch (operator.op()) { - case AND: + return switch (operator.op()) { + case AND -> { for (final ExpressionItem child : enabledChildren) { if (!matchItem(attributeMap, child)) { - return false; + yield false; } } - return true; - case OR: + yield true; + } + case OR -> { for (final ExpressionItem child : enabledChildren) { if (matchItem(attributeMap, child)) { - return true; + yield true; } } - return false; - case NOT: - return enabledChildren.size() == 1 - && !matchItem(attributeMap, enabledChildren.get(0)); - default: - throw new MatchException("Unexpected operator type"); - } + yield false; + } + case NOT -> enabledChildren.size() == 1 + && !matchItem(attributeMap, enabledChildren.get(0)); + }; } } - private boolean matchTerm(final Map attributeMap, final ExpressionTerm term) { - String termField = term.getField(); + private boolean matchTerm(final Map attributeMap, + final ExpressionTerm term) { + // Clean strings to remove unwanted whitespace that the user may have + // added accidentally. + final String termField = NullSafe.trim(term.getField()); final Condition condition = term.getCondition(); String termValue = term.getValue(); final DocRef docRef = term.getDocRef(); - // Clean strings to remove unwanted whitespace that the user may have - // added accidentally. - if (termField != null) { - termField = termField.trim(); - } - if (termValue != null) { - termValue = termValue.trim(); - } - // Try and find the referenced field. - if (termField == null || termField.length() == 0) { + if (NullSafe.isBlankString(termField)) { throw new MatchException("Field not set"); } - final QueryField field = fieldMap.get(termField); + final CIKey termFieldKey = CIKey.of(termField); + final QueryField field = fieldNameToFieldMap.get(termFieldKey); if (field == null) { throw new MatchException("Field not found in index: " + termField); } @@ -152,12 +147,12 @@ private boolean matchTerm(final Map attributeMap, final Expressi throw new MatchException("DocRef not set for field: " + termField); } } else { - if (termValue == null || termValue.length() == 0) { + if (NullSafe.isBlankString(termValue)) { throw new MatchException("Value not set"); } } - final Object attribute = attributeMap.get(term.getField()); + final Object attribute = attributeMap.get(termFieldKey); // Perform null/not null equality if required. if (Condition.IS_NULL.equals(condition)) { @@ -278,27 +273,22 @@ private boolean matchTerm(final Map attributeMap, final Expressi + field.getFldType() + " field type"); } } else { - switch (condition) { - case EQUALS, CONTAINS: - return isStringMatch(termValue, attribute); - case NOT_EQUALS: - return !isStringMatch(termValue, attribute); - case IN: - return isIn(fieldName, termValue, attribute); - case IN_DICTIONARY: - return isInDictionary(fieldName, docRef, field, attribute); - case IN_FOLDER: - return isInFolder(fieldName, docRef, field, attribute); - case IS_DOC_REF: - return isDocRef(fieldName, docRef, field, attribute); - default: { + return switch (condition) { + case EQUALS, CONTAINS -> isStringMatch(termValue, attribute); + case NOT_EQUALS -> !isStringMatch(termValue, attribute); + case IN -> isIn(fieldName, termValue, attribute); + case IN_DICTIONARY -> isInDictionary(fieldName, docRef, field, attribute); + case IN_FOLDER -> isInFolder(fieldName, docRef, field, attribute); + case IS_DOC_REF -> isDocRef(fieldName, docRef, field, attribute); + default -> { if (attribute instanceof final TermMatcher termMatcher) { - return termMatcher.match(field, condition, termValue, docRef); + yield termMatcher.match(field, condition, termValue, docRef); + } else { + throw new MatchException("Unexpected condition '" + condition.getDisplayValue() + "' for " + + field.getFldType() + " field type"); } - throw new MatchException("Unexpected condition '" + condition.getDisplayValue() + "' for " - + field.getFldType() + " field type"); } - } + }; } } @@ -343,7 +333,7 @@ private boolean isIn(final String fieldName, final Object termValue, final Objec } private boolean isStringMatch(final String termValue, final Object attribute) { - final Pattern pattern = patternMap.computeIfAbsent(termValue, t -> + final Pattern pattern = termValueToPatternMap.computeIfAbsent(termValue, t -> Pattern.compile(t.replaceAll("\\*", ".*"), Pattern.CASE_INSENSITIVE)); if (attribute instanceof final DocRef docRef) { @@ -393,7 +383,7 @@ private boolean isInFolder(final String fieldName, final String type = field.getDocRefType(); if (type != null && collectionService != null) { final Set descendants = collectionService.getDescendants(docRef, type); - if (descendants != null && descendants.size() > 0) { + if (NullSafe.hasItems(descendants)) { if (attribute instanceof DocRef) { final String uuid = ((DocRef) attribute).getUuid(); if (uuid != null) { @@ -411,8 +401,10 @@ private boolean isInFolder(final String fieldName, return false; } - private boolean isDocRef(final String fieldName, final DocRef docRef, - final QueryField field, final Object attribute) { + private boolean isDocRef(final String fieldName, + final DocRef docRef, + final QueryField field, + final Object attribute) { if (attribute instanceof DocRef) { final String uuid = ((DocRef) attribute).getUuid(); return (null != uuid && uuid.equals(docRef.getUuid())); diff --git a/stroom-search/stroom-expression-matcher/src/main/java/stroom/expression/matcher/ExpressionMatcherFactory.java b/stroom-search/stroom-expression-matcher/src/main/java/stroom/expression/matcher/ExpressionMatcherFactory.java index 4dea3f66249..43717cce940 100644 --- a/stroom-search/stroom-expression-matcher/src/main/java/stroom/expression/matcher/ExpressionMatcherFactory.java +++ b/stroom-search/stroom-expression-matcher/src/main/java/stroom/expression/matcher/ExpressionMatcherFactory.java @@ -1,9 +1,26 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.expression.matcher; import stroom.collection.api.CollectionService; import stroom.datasource.api.v2.QueryField; import stroom.dictionary.api.WordListProvider; import stroom.expression.api.DateTimeSettings; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; @@ -20,7 +37,7 @@ public class ExpressionMatcherFactory { this.collectionService = collectionService; } - public ExpressionMatcher create(final Map fieldMap) { + public ExpressionMatcher create(final Map fieldMap) { return new ExpressionMatcher(fieldMap, wordListProvider, collectionService, diff --git a/stroom-search/stroom-expression-matcher/src/test/java/stroom/expression/matcher/TestExpressionMatcher.java b/stroom-search/stroom-expression-matcher/src/test/java/stroom/expression/matcher/TestExpressionMatcher.java index 057da067d18..f71ced95f1e 100644 --- a/stroom-search/stroom-expression-matcher/src/test/java/stroom/expression/matcher/TestExpressionMatcher.java +++ b/stroom-search/stroom-expression-matcher/src/test/java/stroom/expression/matcher/TestExpressionMatcher.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ import stroom.query.api.v2.ExpressionOperator.Op; import stroom.query.api.v2.ExpressionTerm; import stroom.query.api.v2.ExpressionTerm.Condition; +import stroom.util.shared.string.CIKey; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.Test; @@ -38,7 +39,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -57,11 +57,9 @@ class TestExpressionMatcher { public static final QueryField FEED = QueryField.createDocRefByUniqueName("Feed", "Feed"); private static final QueryField TYPE = QueryField.createText("Type"); private static final QueryField FRUIT = QueryField.createText("Fruit"); - private static final Map FIELD_MAP = Map.of( - FEED.getFldName(), - FEED, - TYPE.getFldName(), - TYPE); + private static final Map FIELD_MAP = CIKey.mapOf( + FEED.getFldName(), FEED, + TYPE.getFldName(), TYPE); @Test void testSimpleMatch_match() { @@ -168,7 +166,7 @@ void testInDictionary() { .build(); final ExpressionMatcher expressionMatcher = new ExpressionMatcher( - Map.of(FRUIT.getFldName(), FRUIT), + Map.of(CIKey.of(FRUIT.getFldName()), FRUIT), mockWordListProvider, null, DateTimeSettings.builder() @@ -193,7 +191,7 @@ void testInDictionary() { .build(); boolean match = expressionMatcher.match( - Map.of(FRUIT.getFldName(), "orange"), + Map.of(CIKey.of(FRUIT.getFldName()), "orange"), expression); assertThat(match) .isTrue(); @@ -207,7 +205,7 @@ void testInDictionary() { // .isTrue(); } - private void test(final Map attributeMap, + private void test(final Map attributeMap, final ExpressionOperator expression, final boolean outcome) { final ExpressionMatcher expressionMatcher = new ExpressionMatcher( @@ -225,10 +223,9 @@ private ExpressionOperator createExpression(final Op op, final String feedName) return builder.build(); } - private Map createAttributeMap() { - final Map attributeMap = new HashMap<>(); - attributeMap.put(FEED.getFldName(), "TEST_FEED"); - attributeMap.put(TYPE.getFldName(), StreamTypeNames.RAW_EVENTS); - return attributeMap; + private Map createAttributeMap() { + return CIKey.mapOf( + FEED.getFldName(), "TEST_FEED", + TYPE.getFldName(), StreamTypeNames.RAW_EVENTS); } } diff --git a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticClusterStoreImpl.java b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticClusterStoreImpl.java index 8ad9815a362..3259db5c070 100644 --- a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticClusterStoreImpl.java +++ b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticClusterStoreImpl.java @@ -191,8 +191,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticIndexService.java b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticIndexService.java index 404d80d8f2f..00be7bde238 100644 --- a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticIndexService.java +++ b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticIndexService.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.search.elastic; import stroom.datasource.api.v2.QueryField; @@ -5,13 +21,10 @@ import stroom.search.elastic.shared.ElasticIndexField; import java.util.List; -import java.util.Map; public interface ElasticIndexService { List getDataSourceFields(ElasticIndexDoc index); List getFields(ElasticIndexDoc index); - - Map getFieldsMap(ElasticIndexDoc index); } diff --git a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticIndexStoreImpl.java b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticIndexStoreImpl.java index a245edf26f9..15b0f8357e7 100644 --- a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticIndexStoreImpl.java +++ b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/ElasticIndexStoreImpl.java @@ -197,8 +197,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/ElasticSearchProvider.java b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/ElasticSearchProvider.java index 1085f396942..bf87ca2c85f 100644 --- a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/ElasticSearchProvider.java +++ b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/ElasticSearchProvider.java @@ -19,7 +19,6 @@ import stroom.datasource.api.v2.ConditionSet; import stroom.datasource.api.v2.FieldType; import stroom.datasource.api.v2.FindFieldCriteria; -import stroom.datasource.api.v2.IndexField; import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; import stroom.query.api.v2.ExpressionUtil; @@ -30,6 +29,7 @@ import stroom.query.common.v2.CoprocessorsImpl; import stroom.query.common.v2.DataStoreSettings; import stroom.query.common.v2.FieldInfoResultPageBuilder; +import stroom.query.common.v2.IndexFieldMap; import stroom.query.common.v2.IndexFieldProvider; import stroom.query.common.v2.ResultStore; import stroom.query.common.v2.ResultStoreFactory; @@ -49,7 +49,9 @@ import stroom.util.NullSafe; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.CompareUtil; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.ExpandWildcard; @@ -71,14 +73,13 @@ import jakarta.inject.Provider; import java.io.IOException; -import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.TreeMap; import java.util.concurrent.Executor; import java.util.stream.Collectors; @@ -187,11 +188,16 @@ public int getFieldCount(final DocRef docRef) { } @Override - public IndexField getIndexField(final DocRef docRef, final String fieldName) { + public IndexFieldMap getIndexFields(final DocRef docRef, final CIKey fieldName) { final ElasticIndexDoc index = elasticIndexStore.readDocument(docRef); if (index != null) { - final Map indexFieldMap = getFieldsMap(index); - return indexFieldMap.get(fieldName); + final Map> indexFieldMap = getFieldsMap(index); + final Map caseSenseNameToFieldMap = indexFieldMap.get(fieldName); + if (NullSafe.hasEntries(caseSenseNameToFieldMap)) { + return IndexFieldMap.fromFieldsMap(fieldName, caseSenseNameToFieldMap); + } else { + return null; + } } return null; } @@ -318,13 +324,20 @@ private String getAliasPathFromMapping(final String fieldName, final FieldMappin @Override public List getFields(final ElasticIndexDoc index) { - return new ArrayList<>(getFieldsMap(index).values()); + return getFieldsMap(index) + .values() + .stream() + .filter(Objects::nonNull) + .flatMap(map -> + map.values().stream()) + .sorted(CompareUtil.getNullSafeCaseInsensitiveComparator(ElasticIndexField::getFldName)) + .toList(); } - @Override - public Map getFieldsMap(final ElasticIndexDoc index) { + private Map> getFieldsMap(final ElasticIndexDoc index) { final Map fieldMappings = getFieldMappings(index); - final Map fieldsMap = new HashMap<>(); + // Nested map, so we can group fields with the same name, ignoring case + final Map> fieldsMap = new HashMap<>(); fieldMappings.forEach((key, fieldMeta) -> { try { @@ -346,14 +359,17 @@ public Map getFieldsMap(final ElasticIndexDoc index) } final FieldType type = ElasticNativeTypes.fromNativeType(fieldName, nativeType); - fieldsMap.put(fieldName, new ElasticIndexField( + final CIKey fieldNameKey = CIKey.of(fieldName); + final ElasticIndexField elasticIndexField = new ElasticIndexField( null, null, null, fieldName, type, nativeType, - indexed)); + indexed); + fieldsMap.computeIfAbsent(fieldNameKey, k -> new HashMap<>()) + .put(fieldName, elasticIndexField); } catch (UnsupportedTypeException e) { LOGGER.debug(e::getMessage, e); } catch (Exception e) { @@ -376,21 +392,15 @@ private boolean fieldIsIndexed(final FieldMapping field) { final Object mappingInstance = firstFieldMapping.get()._get(); // Detect non-indexed fields for common data types. For all others, assume the field is indexed - if (mappingInstance instanceof KeywordProperty) { - return !Boolean.FALSE.equals(((KeywordProperty) mappingInstance).index()); - } else if (mappingInstance instanceof TextProperty) { - return !Boolean.FALSE.equals(((TextProperty) mappingInstance).index()); - } else if (mappingInstance instanceof BooleanProperty) { - return !Boolean.FALSE.equals(((BooleanProperty) mappingInstance).index()); - } else if (mappingInstance instanceof DateProperty) { - return !Boolean.FALSE.equals(((DateProperty) mappingInstance).index()); - } else if (mappingInstance instanceof NumberPropertyBase) { - return !Boolean.FALSE.equals(((NumberPropertyBase) mappingInstance).index()); - } else if (mappingInstance instanceof IpProperty) { - return !Boolean.FALSE.equals(((IpProperty) mappingInstance).index()); - } else { - return true; - } + return switch (mappingInstance) { + case KeywordProperty keywordProperty -> !Boolean.FALSE.equals(keywordProperty.index()); + case TextProperty textProperty -> !Boolean.FALSE.equals(textProperty.index()); + case BooleanProperty booleanProperty -> !Boolean.FALSE.equals(booleanProperty.index()); + case DateProperty dateProperty -> !Boolean.FALSE.equals(dateProperty.index()); + case NumberPropertyBase numberPropertyBase -> !Boolean.FALSE.equals(numberPropertyBase.index()); + case IpProperty ipProperty -> !Boolean.FALSE.equals(ipProperty.index()); + case null, default -> true; + }; } } catch (Exception e) { return false; @@ -400,7 +410,7 @@ private boolean fieldIsIndexed(final FieldMapping field) { } private Map getFieldMappings(final ElasticIndexDoc elasticIndex) { - Map result = new TreeMap<>(); + Map result = null; if (elasticIndex.getClusterRef() != null) { try { @@ -411,23 +421,16 @@ private Map getFieldMappings(final ElasticIndexDoc elastic LOGGER.error(e::getMessage, e); } } + if (result == null) { + result = Collections.emptyMap(); + } return result; } - private static TreeMap getFlattenedFieldMappings(final ElasticIndexDoc elasticIndex, - final ElasticsearchClient elasticClient) { + private static Map getFlattenedFieldMappings(final ElasticIndexDoc elasticIndex, + final ElasticsearchClient elasticClient) { // Flatten the mappings, which are keyed by index, into a de-duplicated list - final TreeMap mappings = new TreeMap<>((o1, o2) -> { - if (Objects.equals(o1, o2)) { - return 0; - } - if (o2 == null) { - return 1; - } - - return o1.compareToIgnoreCase(o2); - }); - + final Map mappings = new HashMap<>(); final String indexName = elasticIndex.getIndexName(); final GetFieldMappingRequest request = GetFieldMappingRequest.of(r -> r .expandWildcards(ExpandWildcard.Open) @@ -444,8 +447,8 @@ private static TreeMap getFlattenedFieldMappings(final Ela final HashSet multiFieldMappings = new HashSet<>(); allMappings.values().forEach(indexMappings -> indexMappings.mappings().forEach((fieldName, mapping) -> { final Property source = mapping.mapping().get(fieldName); - if (source != null && source._get() instanceof PropertyBase) { - final var multiFields = ((PropertyBase) source._get()).fields(); + if (source != null && source._get() instanceof PropertyBase propertyBase) { + final var multiFields = propertyBase.fields(); if (!multiFields.isEmpty()) { multiFields.forEach((multiFieldName, multiFieldMapping) -> { diff --git a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/ElasticSearchTaskHandler.java b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/ElasticSearchTaskHandler.java index 9d13591d357..0799989ec68 100644 --- a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/ElasticSearchTaskHandler.java +++ b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/ElasticSearchTaskHandler.java @@ -65,7 +65,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; @@ -217,8 +216,8 @@ private void searchSlice(final ElasticIndexDoc elasticIndex, // Limit the returned fields to what the values consumers require final FieldIndex fieldIndex = coprocessors.getFieldIndex(); - final String[] fieldNames = coprocessors.getFieldIndex().getFields(); - searchRequestBuilder.fields(Arrays.stream(fieldNames) + final List fieldNames = coprocessors.getFieldIndex().getFields(); + searchRequestBuilder.fields(fieldNames.stream() .map(fieldName -> FieldAndFormat.of(f -> f.field(fieldName))) .toList() ); diff --git a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/SearchExpressionQueryBuilder.java b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/SearchExpressionQueryBuilder.java index 3c8fe0fc91c..a1836a90693 100644 --- a/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/SearchExpressionQueryBuilder.java +++ b/stroom-search/stroom-search-elastic/src/main/java/stroom/search/elastic/search/SearchExpressionQueryBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.search.elastic.search; @@ -30,6 +29,7 @@ import stroom.query.common.v2.DateExpressionParser; import stroom.query.common.v2.IndexFieldCache; import stroom.search.elastic.shared.ElasticIndexField; +import stroom.util.NullSafe; import stroom.util.functions.TriFunction; import co.elastic.clients.elasticsearch._types.FieldValue; @@ -118,21 +118,14 @@ private Query getQuery(final ExpressionItem item) { } private Query getTermQuery(final ExpressionTerm expressionTerm) { - String field = expressionTerm.getField(); + // Clean strings to remove unwanted whitespace that the user may have added accidentally + final String field = NullSafe.trim(expressionTerm.getField()); final Condition condition = expressionTerm.getCondition(); - String value = expressionTerm.getValue(); + final String value = NullSafe.trim(expressionTerm.getValue()); final DocRef docRef = expressionTerm.getDocRef(); - // Clean strings to remove unwanted whitespace that the user may have added accidentally - if (field != null) { - field = field.trim(); - } - if (value != null) { - value = value.trim(); - } - // Validate the field - if (field == null || field.isEmpty()) { + if (field.isEmpty()) { throw new IllegalArgumentException("Field not set"); } final IndexField indexField = indexFieldCache.get(indexDocRef, field); @@ -142,7 +135,7 @@ private Query getTermQuery(final ExpressionTerm expressionTerm) { final String fieldName = elasticIndexField.getFldName(); // Validate the expression - if (value == null || value.isEmpty()) { + if (value.isEmpty()) { return null; } if (Condition.IN_DICTIONARY.equals(condition)) { @@ -182,10 +175,10 @@ private Query getTermQuery(final ExpressionTerm expressionTerm) { } private Query buildStringQuery(final Condition condition, - final String expression, - final DocRef docRef, - final ElasticIndexField indexField, - final String fieldName) { + final String expression, + final DocRef docRef, + final ElasticIndexField indexField, + final String fieldName) { final List terms = tokenizeExpression(expression).filter(term -> !term.isEmpty()).toList(); if (terms.isEmpty()) { return null; diff --git a/stroom-search/stroom-search-extraction/src/main/java/stroom/search/extraction/ExtractionDecorator.java b/stroom-search/stroom-search-extraction/src/main/java/stroom/search/extraction/ExtractionDecorator.java index 2bb336e34f9..20c72d88848 100644 --- a/stroom-search/stroom-search-extraction/src/main/java/stroom/search/extraction/ExtractionDecorator.java +++ b/stroom-search/stroom-search-extraction/src/main/java/stroom/search/extraction/ExtractionDecorator.java @@ -1,8 +1,23 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.search.extraction; import stroom.data.store.api.DataException; import stroom.docref.DocRef; -import stroom.index.shared.IndexConstants; import stroom.meta.api.MetaService; import stroom.meta.shared.Meta; import stroom.pipeline.PipelineStore; @@ -30,6 +45,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.pipeline.scope.PipelineScopeRunnable; +import stroom.util.shared.query.FieldNames; import jakarta.inject.Provider; @@ -118,8 +134,8 @@ public StoredDataQueue createStoredDataQueue(final Coprocessors coprocessors, this.dataSource = query.getDataSource(); // We are going to do extraction or at least filter streams so add fields to the field index to do this. - coprocessors.getFieldIndex().create(IndexConstants.STREAM_ID); - coprocessors.getFieldIndex().create(IndexConstants.EVENT_ID); + coprocessors.getFieldIndex().create(FieldNames.FALLBACK_STREAM_ID_FIELD_KEY); + coprocessors.getFieldIndex().create(FieldNames.FALLBACK_EVENT_ID_FIELD_KEY); coprocessors.forEachExtractionCoprocessor((docRef, coprocessorSet) -> { // We assume all coprocessors for the same extraction use the same field index map. // This is only the case at the moment as the CoprocessorsFactory creates field index maps this way. diff --git a/stroom-search/stroom-search-impl/src/main/java/stroom/search/impl/SearchException.java b/stroom-search/stroom-search-impl/src/main/java/stroom/search/impl/SearchException.java index d30e714ab70..8c4bfe4653e 100644 --- a/stroom-search/stroom-search-impl/src/main/java/stroom/search/impl/SearchException.java +++ b/stroom-search/stroom-search-impl/src/main/java/stroom/search/impl/SearchException.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2016-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,8 @@ package stroom.search.impl; +import stroom.util.NullSafe; + public class SearchException extends RuntimeException { private static final long serialVersionUID = -482925256715483280L; @@ -33,7 +35,7 @@ public static SearchException wrap(final Throwable t) { return (SearchException) t; } String message = t.getMessage(); - if (message == null || message.length() == 0) { + if (NullSafe.isEmptyString(message)) { message = t.toString(); } return new SearchException(message, t); diff --git a/stroom-search/stroom-search-impl/src/main/java/stroom/search/impl/SearchModule.java b/stroom-search/stroom-search-impl/src/main/java/stroom/search/impl/SearchModule.java index 958f358b92c..f4fdae46578 100644 --- a/stroom-search/stroom-search-impl/src/main/java/stroom/search/impl/SearchModule.java +++ b/stroom-search/stroom-search-impl/src/main/java/stroom/search/impl/SearchModule.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 Crown Copyright + * Copyright 2018-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -84,6 +84,10 @@ public int hashCode() { return 0; } + + // -------------------------------------------------------------------------------- + + private static class EvictExpiredElements extends RunnableWrapper { @Inject diff --git a/stroom-search/stroom-search-solr/src/main/java/stroom/search/solr/SolrIndexStoreImpl.java b/stroom-search/stroom-search-solr/src/main/java/stroom/search/solr/SolrIndexStoreImpl.java index 18dd1be0517..947ba3993f8 100644 --- a/stroom-search/stroom-search-solr/src/main/java/stroom/search/solr/SolrIndexStoreImpl.java +++ b/stroom-search/stroom-search-solr/src/main/java/stroom/search/solr/SolrIndexStoreImpl.java @@ -435,8 +435,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-search/stroom-search-solr/src/main/java/stroom/search/solr/search/SolrSearchProvider.java b/stroom-search/stroom-search-solr/src/main/java/stroom/search/solr/search/SolrSearchProvider.java index 8133c8b0425..0b75be87807 100644 --- a/stroom-search/stroom-search-solr/src/main/java/stroom/search/solr/search/SolrSearchProvider.java +++ b/stroom-search/stroom-search-solr/src/main/java/stroom/search/solr/search/SolrSearchProvider.java @@ -32,6 +32,7 @@ import stroom.query.common.v2.DataStoreSettings; import stroom.query.common.v2.FieldInfoResultPageBuilder; import stroom.query.common.v2.IndexFieldCache; +import stroom.query.common.v2.IndexFieldMap; import stroom.query.common.v2.IndexFieldProvider; import stroom.query.common.v2.ResultStore; import stroom.query.common.v2.ResultStoreFactory; @@ -40,10 +41,10 @@ import stroom.search.solr.search.SearchExpressionQueryBuilder.SearchExpressionQuery; import stroom.search.solr.shared.SolrIndexDataSourceFieldUtil; import stroom.search.solr.shared.SolrIndexDoc; -import stroom.search.solr.shared.SolrIndexField; import stroom.security.api.SecurityContext; import stroom.util.NullSafe; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import org.slf4j.Logger; @@ -51,9 +52,11 @@ import java.util.Collections; import java.util.List; -import java.util.Objects; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; // used by DI @SuppressWarnings("unused") @@ -118,17 +121,25 @@ public int getFieldCount(final DocRef docRef) { } @Override - public IndexField getIndexField(final DocRef docRef, final String fieldName) { + public IndexFieldMap getIndexFields(final DocRef docRef, final CIKey fieldName) { final SolrIndexDoc index = solrIndexStore.readDocument(docRef); - if (index != null && index.getFields() != null) { - final Optional optionalSolrIndexField = index + if (NullSafe.nonNull(index, SolrIndexDoc::getFields)) { + final Map fieldMap = index .getFields() .stream() - .filter(field -> Objects.equals(fieldName, field.getFldName())) - .findFirst(); - return optionalSolrIndexField.orElse(null); + .filter(field -> + CIKey.equalsIgnoreCase(fieldName, field.getFldName())) + .map(solrIndexField -> (IndexField) solrIndexField) + .collect(Collectors.toMap(IndexField::getFldName, Function.identity())); + + if (NullSafe.hasEntries(fieldMap)) { + return IndexFieldMap.fromFieldsMap(fieldName, fieldMap); + } else { + return null; + } + } else { + return null; } - return null; } @Override diff --git a/stroom-security/stroom-security-api/src/main/java/stroom/security/api/UserIdentityFactory.java b/stroom-security/stroom-security-api/src/main/java/stroom/security/api/UserIdentityFactory.java index 7a5c214e81a..f3bd0ce8a4e 100644 --- a/stroom-security/stroom-security-api/src/main/java/stroom/security/api/UserIdentityFactory.java +++ b/stroom-security/stroom-security-api/src/main/java/stroom/security/api/UserIdentityFactory.java @@ -1,5 +1,23 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.api; +import stroom.util.shared.string.CIKey; + import jakarta.servlet.http.HttpServletRequest; import java.util.Map; @@ -39,7 +57,7 @@ public interface UserIdentityFactory { /** * Remove any authentication headers key/value pairs from the map */ - void removeAuthEntries(final Map headers); + void removeAuthEntries(final Map headers); /** * @return The authentication/authorisation headers to enable authentication with this user diff --git a/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/AbstractUserIdentityFactory.java b/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/AbstractUserIdentityFactory.java index 83981620168..af9eaade0ae 100644 --- a/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/AbstractUserIdentityFactory.java +++ b/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/AbstractUserIdentityFactory.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.common.impl; import stroom.security.api.HasJwt; @@ -21,6 +37,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -152,7 +169,7 @@ public boolean hasAuthenticationCertificate(final HttpServletRequest request) { } @Override - public void removeAuthEntries(final Map headers) { + public void removeAuthEntries(final Map headers) { jwtContextFactory.removeAuthorisationEntries(headers); } diff --git a/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/JwtContextFactory.java b/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/JwtContextFactory.java index 468ef678d39..b0530c77a2c 100644 --- a/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/JwtContextFactory.java +++ b/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/JwtContextFactory.java @@ -1,5 +1,23 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.common.impl; +import stroom.util.shared.string.CIKey; + import jakarta.servlet.http.HttpServletRequest; import org.jose4j.jwt.consumer.JwtContext; @@ -10,7 +28,7 @@ public interface JwtContextFactory { boolean hasToken(HttpServletRequest request); - void removeAuthorisationEntries(final Map headers); + void removeAuthorisationEntries(final Map headers); Map createAuthorisationEntries(final String accessToken); diff --git a/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/JwtUtil.java b/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/JwtUtil.java index a3712c49d03..ee1a23fe212 100644 --- a/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/JwtUtil.java +++ b/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/JwtUtil.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.common.impl; import stroom.security.openid.api.OpenId; @@ -7,6 +23,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; import jakarta.servlet.http.HttpServletRequest; import org.jose4j.json.internal.json_simple.parser.ContainerFactory; @@ -46,6 +63,14 @@ public Map createObjectContainer() { private JwtUtil() { } + /** + * Get the JSON Web Signature from the specified request header + */ + public static Optional getJwsFromHeader(final HttpServletRequest request, + final CIKey headerName) { + return getJwsFromHeader(request, headerName.get()); + } + /** * Get the JSON Web Signature from the specified request header */ @@ -104,7 +129,7 @@ public static String getUniqueIdentity(final OpenIdConfiguration openIdConfigura * Maps to the 'name' column in stroom_user table. */ public static Optional getUserDisplayName(final OpenIdConfiguration openIdConfiguration, - final JwtClaims jwtClaims) { + final JwtClaims jwtClaims) { Objects.requireNonNull(openIdConfiguration); Objects.requireNonNull(jwtClaims); final String userDisplayNameClaim = openIdConfiguration.getUserDisplayNameClaim(); diff --git a/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/StandardJwtContextFactory.java b/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/StandardJwtContextFactory.java index e41dc570fec..034a8318838 100644 --- a/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/StandardJwtContextFactory.java +++ b/stroom-security/stroom-security-common-impl/src/main/java/stroom/security/common/impl/StandardJwtContextFactory.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.common.impl; import stroom.security.api.exception.AuthenticationException; @@ -13,6 +29,8 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; @@ -78,21 +96,22 @@ public class StandardJwtContextFactory implements JwtContextFactory { /** * The access token from the token endpoint, in plain text. */ - static final String AMZN_OIDC_ACCESS_TOKEN_HEADER = "x-amzn-oidc-accesstoken"; + static final CIKey AMZN_OIDC_ACCESS_TOKEN_HEADER_KEY = CIKey.ofStaticKey("x-amzn-oidc-accesstoken"); /** * The subject field (sub) from the user info endpoint, in plain text. */ - static final String AMZN_OIDC_IDENTITY_HEADER = "x-amzn-oidc-identity"; + static final CIKey AMZN_OIDC_IDENTITY_HEADER_KEY = CIKey.ofStaticKey("x-amzn-oidc-identity"); /** * The user claims, in JSON web tokens (JWT) format. */ - static final String AMZN_OIDC_DATA_HEADER = "x-amzn-oidc-data"; + static final CIKey AMZN_OIDC_DATA_HEADER_KEY = CIKey.ofStaticKey("x-amzn-oidc-data"); + static final String AMZN_OIDC_SIGNER_PREFIX = "arn:"; - static final String SIGNER_HEADER_KEY = "signer"; + static final String SIGNER_HEADER_JSON_KEY = "signer"; static final String AMZN_OIDC_SIGNER_SPLIT_CHAR = ":"; static final Pattern AMZN_REGION_PATTERN = Pattern.compile("^[a-z0-9-]+$"); - private static final String AUTHORIZATION_HEADER = HttpHeaders.AUTHORIZATION; + private static final CIKey AUTHORIZATION_HEADER_KEY = CIKeys.AUTHORIZATION; private final Provider openIdConfigurationProvider; private final OpenIdPublicKeysSupplier openIdPublicKeysSupplier; @@ -146,12 +165,12 @@ public boolean hasToken(final HttpServletRequest request) { } @Override - public void removeAuthorisationEntries(final Map headers) { + public void removeAuthorisationEntries(final Map headers) { if (NullSafe.hasEntries(headers)) { - headers.remove(AUTHORIZATION_HEADER); - headers.remove(AMZN_OIDC_ACCESS_TOKEN_HEADER); - headers.remove(AMZN_OIDC_DATA_HEADER); - headers.remove(AMZN_OIDC_IDENTITY_HEADER); + headers.remove(CIKeys.AUTHORIZATION); + headers.remove(AMZN_OIDC_ACCESS_TOKEN_HEADER_KEY); + headers.remove(AMZN_OIDC_DATA_HEADER_KEY); + headers.remove(AMZN_OIDC_IDENTITY_HEADER_KEY); } } @@ -189,10 +208,11 @@ public Optional getJwtContext(final HttpServletRequest request) { if (LOGGER.isTraceEnabled()) { // Only output non-null ones. Probably the only useful one is the ID one as the rest are base64 encoded final String headers = Stream.of( - AUTHORIZATION_HEADER, - AMZN_OIDC_ACCESS_TOKEN_HEADER, - AMZN_OIDC_IDENTITY_HEADER, - AMZN_OIDC_DATA_HEADER) + AUTHORIZATION_HEADER_KEY, + AMZN_OIDC_ACCESS_TOKEN_HEADER_KEY, + AMZN_OIDC_IDENTITY_HEADER_KEY, + AMZN_OIDC_DATA_HEADER_KEY) + .map(CIKey::get) .map(key -> new SimpleEntry<>(key, request.getHeader(key))) .filter(entry -> entry.getValue() != null) .map(keyValue -> "\n " + keyValue.getKey() + ": '" + keyValue.getValue() + "'") @@ -210,8 +230,8 @@ public Optional getJwtContext(final HttpServletRequest request) { } private boolean isAwsSignedToken(final HeaderToken headerToken, final JwsParts jwsParts) { - if (AMZN_OIDC_DATA_HEADER.equals(headerToken.header)) { - LOGGER.debug("Found header {}", AMZN_OIDC_DATA_HEADER); + if (AMZN_OIDC_DATA_HEADER_KEY.equals(headerToken.header)) { + LOGGER.debug("Found header {}", AMZN_OIDC_DATA_HEADER_KEY); // Request came from an AWS load balancer that did the auth flow return true; } else { @@ -224,9 +244,9 @@ private boolean isAwsSignedToken(final HeaderToken headerToken, final JwsParts j private boolean isAwsSignedToken(final JwsParts jwsParts) { // Request may have come from another stroom node that has passed on an AWS signed // token, but not in the special AWS header - return jwsParts.getHeaderValue(SIGNER_HEADER_KEY) + return jwsParts.getHeaderValue(SIGNER_HEADER_JSON_KEY) .filter(val -> { - LOGGER.debug("{} is {}", SIGNER_HEADER_KEY, val); + LOGGER.debug("{} is {}", SIGNER_HEADER_JSON_KEY, val); return val.startsWith(AMZN_OIDC_SIGNER_PREFIX); }) .isPresent(); @@ -264,13 +284,13 @@ private Optional getTokenFromHeader(final HttpServletRequest reques // We may be dealing with requests of either form if (LOGGER.isDebugEnabled()) { // This will log the AWS identity if there is one - JwtUtil.getJwsFromHeader(request, AMZN_OIDC_IDENTITY_HEADER); + JwtUtil.getJwsFromHeader(request, AMZN_OIDC_IDENTITY_HEADER_KEY); } - return JwtUtil.getJwsFromHeader(request, AMZN_OIDC_DATA_HEADER) - .map(jws -> new HeaderToken(AMZN_OIDC_DATA_HEADER, jws)) + return JwtUtil.getJwsFromHeader(request, AMZN_OIDC_DATA_HEADER_KEY) + .map(jws -> new HeaderToken(AMZN_OIDC_DATA_HEADER_KEY, jws)) .or(() -> - JwtUtil.getJwsFromHeader(request, AUTHORIZATION_HEADER) - .map(jws -> new HeaderToken(AUTHORIZATION_HEADER, jws))); + JwtUtil.getJwsFromHeader(request, AUTHORIZATION_HEADER_KEY) + .map(jws -> new HeaderToken(AUTHORIZATION_HEADER_KEY, jws))); } @@ -475,12 +495,12 @@ static String getAwsPublicKeyUri(final JwsParts jwsParts, final Set expectedSignerPrefixes) { final Map headerValues = jwsParts.getHeaderValues( - SIGNER_HEADER_KEY, + SIGNER_HEADER_JSON_KEY, OpenId.KEY_ID); - final String signer = Optional.ofNullable(headerValues.get(SIGNER_HEADER_KEY)) + final String signer = Optional.ofNullable(headerValues.get(SIGNER_HEADER_JSON_KEY)) .orElseThrow(() -> new RuntimeException(LogUtil.message("Missing '{}' key in jws header {}", - SIGNER_HEADER_KEY, jwsParts.header))); + SIGNER_HEADER_JSON_KEY, jwsParts.header))); // 'arn:aws:elasticloadbalancing:region-code:account-id:loadbalancer/app/load-balancer-name/load-balancer-id' // The LB ID is not known at provisioning time so validate signer against a set of valid prefixes @@ -501,7 +521,7 @@ static String getAwsPublicKeyUri(final JwsParts jwsParts, "configuration property: [{}]. You need to set '{}' to the partial ARN(s) of the " + "AWS load balancer that is handling authentication for Stroom. The partial " + "ARN needs to include at least up to the account ID part.", - SIGNER_HEADER_KEY, + SIGNER_HEADER_JSON_KEY, signer, AbstractOpenIdConfig.PROP_NAME_EXPECTED_SIGNER_PREFIXES, String.join(", ", NullSafe.set(expectedSignerPrefixes)), @@ -529,7 +549,7 @@ private static String extractAwsRegionFromSigner(final String signer) { final String[] signerParts = signer.split(AMZN_OIDC_SIGNER_SPLIT_CHAR); if (signerParts.length < 4) { throw new RuntimeException(LogUtil.message("Unable to parse value for '{}' key in JWS header {}", - SIGNER_HEADER_KEY, signer)); + SIGNER_HEADER_JSON_KEY, signer)); } final String awsRegion = signerParts[3]; @@ -602,7 +622,7 @@ private PublicKey decodePublicKey(final String pem, final String alg) { private record HeaderToken( - String header, + CIKey header, String jwt) { } @@ -614,7 +634,7 @@ private record HeaderToken( // Pkg private for testing record JwsParts( String jws, - String header, + String header, // base64 encoded json String payload, String signature) { diff --git a/stroom-security/stroom-security-common-impl/src/test/java/stroom/security/common/impl/TestStandardJwtContextFactory.java b/stroom-security/stroom-security-common-impl/src/test/java/stroom/security/common/impl/TestStandardJwtContextFactory.java index 8e8d8c5363f..ae8cc8e5399 100644 --- a/stroom-security/stroom-security-common-impl/src/test/java/stroom/security/common/impl/TestStandardJwtContextFactory.java +++ b/stroom-security/stroom-security-common-impl/src/test/java/stroom/security/common/impl/TestStandardJwtContextFactory.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.common.impl; import stroom.security.common.impl.StandardJwtContextFactory.JwsParts; @@ -239,7 +255,7 @@ void getAwsPublicKeyUriFromSigner_signerNotFound() { StandardJwtContextFactory.getAwsPublicKeyUri(jwsParts, Set.of(signer)); }) .hasMessageContaining("Missing") - .hasMessageContaining(StandardJwtContextFactory.SIGNER_HEADER_KEY) + .hasMessageContaining(StandardJwtContextFactory.SIGNER_HEADER_JSON_KEY) .isInstanceOf(RuntimeException.class); } diff --git a/stroom-security/stroom-security-identity-db/src/main/java/stroom/security/identity/db/AccountDaoImpl.java b/stroom-security/stroom-security-identity-db/src/main/java/stroom/security/identity/db/AccountDaoImpl.java index e777bd3a046..a71a4452e08 100644 --- a/stroom-security/stroom-security-identity-db/src/main/java/stroom/security/identity/db/AccountDaoImpl.java +++ b/stroom-security/stroom-security-identity-db/src/main/java/stroom/security/identity/db/AccountDaoImpl.java @@ -1,19 +1,17 @@ /* + * Copyright 2017-2024 Crown Copyright * - * Copyright 2017 Crown Copyright + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * http://www.apache.org/licenses/LICENSE-2.0 * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package stroom.security.identity.db; @@ -36,6 +34,7 @@ import stroom.util.logging.LambdaLoggerFactory; import stroom.util.shared.CompareUtil; import stroom.util.shared.PageResponse; +import stroom.util.shared.string.CIKey; import com.google.common.base.Strings; import jakarta.inject.Inject; @@ -140,7 +139,7 @@ class AccountDaoImpl implements AccountDao { protected static final String FIELD_NAME_LAST_LOGIN_MS = "lastLoginMs"; protected static final String FIELD_NAME_LOGIN_FAILURES = "loginFailures"; protected static final String FIELD_NAME_COMMENTS = "comments"; - private static final Map> FIELD_MAP = Map.ofEntries( + private static final Map> FIELD_MAP = CIKey.mapOfEntries( entry("id", ACCOUNT.ID), entry("version", ACCOUNT.VERSION), entry("createTimeMs", ACCOUNT.CREATE_TIME_MS), diff --git a/stroom-security/stroom-security-impl-db/src/main/java/stroom/security/impl/db/ApiKeyDaoImpl.java b/stroom-security/stroom-security-impl-db/src/main/java/stroom/security/impl/db/ApiKeyDaoImpl.java index 12a65ff10eb..c1c22fb46c2 100644 --- a/stroom-security/stroom-security-impl-db/src/main/java/stroom/security/impl/db/ApiKeyDaoImpl.java +++ b/stroom-security/stroom-security-impl-db/src/main/java/stroom/security/impl/db/ApiKeyDaoImpl.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.impl.db; import stroom.db.util.GenericDao; @@ -20,6 +36,7 @@ import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; import stroom.util.shared.UserRef; +import stroom.util.shared.string.CIKey; import com.mysql.cj.exceptions.MysqlErrorNumbers; import jakarta.inject.Inject; @@ -54,7 +71,7 @@ public class ApiKeyDaoImpl implements ApiKeyDao { private static final Field OWNER_UI_VALUE_JOOQ_FIELD = DSL.field( DSL.coalesce(STROOM_USER.DISPLAY_NAME, STROOM_USER.NAME).as(OWNER_UI_VALUE_JOOQ_FIELD_NAME)); - private static final Map> FIELD_MAP = Map.of( + private static final Map> FIELD_MAP = CIKey.mapOf( FindApiKeyCriteria.FIELD_NAME, API_KEY.NAME, FindApiKeyCriteria.FIELD_PREFIX, API_KEY.API_KEY_PREFIX, FindApiKeyCriteria.FIELD_OWNER, OWNER_UI_VALUE_JOOQ_FIELD, diff --git a/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/DelegatingJwtContextFactory.java b/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/DelegatingJwtContextFactory.java index c4b0587adf3..0ffac0266d6 100644 --- a/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/DelegatingJwtContextFactory.java +++ b/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/DelegatingJwtContextFactory.java @@ -1,9 +1,26 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.impl; import stroom.security.common.impl.JwtContextFactory; import stroom.security.common.impl.StandardJwtContextFactory; import stroom.security.openid.api.IdpType; import stroom.util.NullSafe; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -19,6 +36,7 @@ // * user request which always uses the internal idp. It also takes into account whether // * an external IDP is in use. // */ + /** * A front for {@link InternalJwtContextFactory} and {@link StandardJwtContextFactory} * that picks the delegate based on the identity provider that has been configured, e.g. @@ -45,7 +63,7 @@ public boolean hasToken(final HttpServletRequest request) { } @Override - public void removeAuthorisationEntries(final Map headers) { + public void removeAuthorisationEntries(final Map headers) { if (NullSafe.hasEntries(headers)) { getDelegate().removeAuthorisationEntries(headers); } @@ -73,10 +91,8 @@ public Optional getJwtContext(final String jwt, final boolean doVeri private JwtContextFactory getDelegate() { return switch (openIdConfigProvider.get().getIdentityProviderType()) { - case INTERNAL_IDP, TEST_CREDENTIALS -> - internalJwtContextFactory; - case EXTERNAL_IDP -> - standardJwtContextFactory; + case INTERNAL_IDP, TEST_CREDENTIALS -> internalJwtContextFactory; + case EXTERNAL_IDP -> standardJwtContextFactory; case NO_IDP -> throw new UnsupportedOperationException("No JwtContextFactory when IDP type is " + IdpType.NO_IDP); }; diff --git a/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/DocPermissionResourceImpl.java b/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/DocPermissionResourceImpl.java index 5d61b7f2288..efbf2246b18 100644 --- a/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/DocPermissionResourceImpl.java +++ b/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/DocPermissionResourceImpl.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.impl; import stroom.docref.DocRef; diff --git a/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/InternalJwtContextFactory.java b/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/InternalJwtContextFactory.java index 2f0ab63d3a1..be8edecb7d8 100644 --- a/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/InternalJwtContextFactory.java +++ b/stroom-security/stroom-security-impl/src/main/java/stroom/security/impl/InternalJwtContextFactory.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.impl; import stroom.security.common.impl.JwtContextFactory; @@ -9,6 +25,8 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -36,7 +54,8 @@ class InternalJwtContextFactory implements JwtContextFactory { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(InternalJwtContextFactory.class); - private static final String AUTHORIZATION_HEADER = "Authorization"; + private static final CIKey AUTHORIZATION_HEADER_KEY = CIKeys.AUTHORIZATION; + private static final String AUTHORIZATION_HEADER = AUTHORIZATION_HEADER_KEY.get(); private final OpenIdClientFactory openIdClientDetailsFactory; private final PublicJsonWebKeyProvider publicJsonWebKeyProvider; @@ -58,9 +77,9 @@ public boolean hasToken(final HttpServletRequest request) { } @Override - public void removeAuthorisationEntries(final Map headers) { + public void removeAuthorisationEntries(final Map headers) { if (NullSafe.hasEntries(headers)) { - headers.remove(AUTHORIZATION_HEADER); + headers.remove(AUTHORIZATION_HEADER_KEY); } } diff --git a/stroom-security/stroom-security-mock/src/main/java/stroom/security/mock/MockUserIdentityFactory.java b/stroom-security/stroom-security-mock/src/main/java/stroom/security/mock/MockUserIdentityFactory.java index 02650bd3e0f..8819b988b1c 100644 --- a/stroom-security/stroom-security-mock/src/main/java/stroom/security/mock/MockUserIdentityFactory.java +++ b/stroom-security/stroom-security-mock/src/main/java/stroom/security/mock/MockUserIdentityFactory.java @@ -1,8 +1,25 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.security.mock; import stroom.security.api.ServiceUserFactory; import stroom.security.api.UserIdentity; import stroom.security.api.UserIdentityFactory; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import jakarta.servlet.http.HttpServletRequest; @@ -38,7 +55,7 @@ public boolean isServiceUser(final UserIdentity userIdentity) { } @Override - public void removeAuthEntries(final Map headers) { + public void removeAuthEntries(final Map headers) { } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/ScyllaDbDocStoreImpl.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/ScyllaDbDocStoreImpl.java index cbbfa7b29d2..754469d3439 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/ScyllaDbDocStoreImpl.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/ScyllaDbDocStoreImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.state.impl; @@ -236,8 +235,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/ScyllaDbExpressionUtil.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/ScyllaDbExpressionUtil.java index 04a53be3be3..83f960ae738 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/ScyllaDbExpressionUtil.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/ScyllaDbExpressionUtil.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl; import stroom.expression.api.DateTimeSettings; @@ -8,6 +24,7 @@ import stroom.state.impl.dao.ScyllaDbColumn; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; import com.datastax.oss.driver.api.core.type.DataTypes; import com.datastax.oss.driver.api.querybuilder.Literal; @@ -25,7 +42,7 @@ public class ScyllaDbExpressionUtil { private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(ScyllaDbExpressionUtil.class); - public static void getRelations(final Map columnMap, + public static void getRelations(final Map columnMap, final ExpressionOperator expressionOperator, final List relations, final DateTimeSettings dateTimeSettings) { @@ -55,14 +72,14 @@ public static void getRelations(final Map columnMap, } } - private static Relation convertTerm(final Map columnMap, + private static Relation convertTerm(final Map columnMap, final ExpressionTerm term, final DateTimeSettings dateTimeSettings) { - String field = term.getField(); - Condition condition = term.getCondition(); + final String field = term.getField(); + final Condition condition = term.getCondition(); String value = term.getValue(); - final ScyllaDbColumn column = columnMap.get(field); + final ScyllaDbColumn column = columnMap.get(CIKey.of(field)); if (column == null) { throw new RuntimeException("Unexpected column " + field); } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/StateDocStoreImpl.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/StateDocStoreImpl.java index 6af0dfb2a5e..103f205578a 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/StateDocStoreImpl.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/StateDocStoreImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package stroom.state.impl; @@ -315,8 +314,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/StateSearchProvider.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/StateSearchProvider.java index a04ae9009da..dc89002ad2f 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/StateSearchProvider.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/StateSearchProvider.java @@ -17,7 +17,6 @@ package stroom.state.impl; import stroom.datasource.api.v2.FindFieldCriteria; -import stroom.datasource.api.v2.IndexField; import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; import stroom.entity.shared.ExpressionCriteria; @@ -31,6 +30,7 @@ import stroom.query.common.v2.CoprocessorsImpl; import stroom.query.common.v2.DataStoreSettings; import stroom.query.common.v2.FieldInfoResultPageBuilder; +import stroom.query.common.v2.IndexFieldMap; import stroom.query.common.v2.IndexFieldProvider; import stroom.query.common.v2.ResultStore; import stroom.query.common.v2.ResultStoreFactory; @@ -47,6 +47,7 @@ import stroom.util.logging.LambdaLoggerFactory; import stroom.util.logging.LogUtil; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import com.datastax.oss.driver.api.core.CqlSession; import jakarta.inject.Inject; @@ -125,14 +126,21 @@ public int getFieldCount(final DocRef docRef) { } @Override - public IndexField getIndexField(final DocRef docRef, final String fieldName) { + public IndexFieldMap getIndexFields(final DocRef docRef, final CIKey fieldName) { final StateDoc doc = getStateDoc(docRef); - final Map fieldMap = StateFieldUtil.getFieldMap(doc.getStateType()); + final Map fieldMap = StateFieldUtil.getFieldMap(doc.getStateType()); final QueryField queryField = fieldMap.get(fieldName); + if (queryField == null) { return null; + } else { + final IndexFieldImpl indexField = IndexFieldImpl.builder() + .fldName(queryField.getFldName()) + .fldType(queryField.getFldType()) + .build(); + + return IndexFieldMap.forSingleField(fieldName, indexField); } - return IndexFieldImpl.builder().fldName(queryField.getFldName()).fldType(queryField.getFldType()).build(); } @Override diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/RangedStateDao.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/RangedStateDao.java index a9db0de39e0..e69343ad064 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/RangedStateDao.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/RangedStateDao.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.entity.shared.ExpressionCriteria; @@ -6,6 +22,8 @@ import stroom.query.language.functions.ValuesConsumer; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; @@ -40,17 +58,12 @@ public class RangedStateDao extends AbstractStateDao { private static final CqlIdentifier COLUMN_VALUE = CqlIdentifier.fromCql("value"); private static final CqlIdentifier COLUMN_INSERT_TIME = CqlIdentifier.fromCql("insert_time"); - private static final Map COLUMN_MAP = Map.of( - RangedStateFields.KEY_START, - new ScyllaDbColumn(RangedStateFields.KEY_START, DataTypes.BIGINT, COLUMN_KEY_START), - RangedStateFields.KEY_END, - new ScyllaDbColumn(RangedStateFields.KEY_END, DataTypes.BIGINT, COLUMN_KEY_END), - RangedStateFields.VALUE_TYPE, - new ScyllaDbColumn(RangedStateFields.VALUE_TYPE, DataTypes.TINYINT, COLUMN_VALUE_TYPE), - RangedStateFields.VALUE, - new ScyllaDbColumn(RangedStateFields.VALUE, DataTypes.BLOB, COLUMN_VALUE), - RangedStateFields.INSERT_TIME, - new ScyllaDbColumn(RangedStateFields.INSERT_TIME, DataTypes.TIMESTAMP, COLUMN_INSERT_TIME)); + private static final Map FIELD_NAME_TO_COLUMN_MAP = Map.ofEntries( + StateFieldUtil.createNameToColumnEntry(CIKeys.KEY_START, DataTypes.BIGINT, COLUMN_KEY_START), + StateFieldUtil.createNameToColumnEntry(CIKeys.KEY_END, DataTypes.BIGINT, COLUMN_KEY_END), + StateFieldUtil.createNameToColumnEntry(CIKeys.VALUE_TYPE, DataTypes.TINYINT, COLUMN_VALUE_TYPE), + StateFieldUtil.createNameToColumnEntry(CIKeys.VALUE, DataTypes.BLOB, COLUMN_VALUE), + StateFieldUtil.createNameToColumnEntry(CIKeys.INSERT_TIME, DataTypes.TIMESTAMP, COLUMN_INSERT_TIME)); public RangedStateDao(final Provider sessionProvider, final String tableName) { super(sessionProvider, CqlIdentifier.fromCql(tableName)); @@ -138,9 +151,9 @@ public void search(final ExpressionCriteria criteria, final SearchHelper searchHelper = new SearchHelper( sessionProvider, table, - COLUMN_MAP, - RangedStateFields.VALUE_TYPE, - RangedStateFields.VALUE); + FIELD_NAME_TO_COLUMN_MAP, + CIKeys.VALUE_TYPE, + CIKeys.VALUE); searchHelper.search(criteria, fieldIndex, dateTimeSettings, consumer); } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/RangedStateFields.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/RangedStateFields.java index 0656c37f887..40b654d0f61 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/RangedStateFields.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/RangedStateFields.java @@ -1,35 +1,47 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; -import java.util.Arrays; import java.util.List; import java.util.Map; public interface RangedStateFields { - String KEY_START = "KeyStart"; - String KEY_END = "KeyEnd"; - String VALUE_TYPE = "ValueType"; - String VALUE = "Value"; - String INSERT_TIME = "InsertTime"; - QueryField KEY_START_FIELD = QueryField.createLong(KEY_START); - QueryField KEY_END_FIELD = QueryField.createText(KEY_END); - QueryField VALUE_TYPE_FIELD = QueryField.createText(VALUE_TYPE, false); - QueryField VALUE_FIELD = QueryField.createText(VALUE, false); - QueryField INSERT_TIME_FIELD = QueryField.createText(INSERT_TIME, false); + QueryField KEY_START_FIELD = QueryField.createLong(CIKeys.KEY_START, true); + QueryField KEY_END_FIELD = QueryField.createText(CIKeys.KEY_END, true); + QueryField VALUE_TYPE_FIELD = QueryField.createText(CIKeys.VALUE_TYPE, false); + QueryField VALUE_FIELD = QueryField.createText(CIKeys.VALUE, false); + QueryField INSERT_TIME_FIELD = QueryField.createText(CIKeys.INSERT_TIME, false); - List FIELDS = Arrays.asList( + List FIELDS = List.of( KEY_START_FIELD, KEY_END_FIELD, VALUE_TYPE_FIELD, VALUE_FIELD, INSERT_TIME_FIELD); - Map FIELD_MAP = Map.of( - KEY_START, KEY_START_FIELD, - KEY_END, KEY_END_FIELD, - VALUE_TYPE, VALUE_TYPE_FIELD, - VALUE, VALUE_FIELD, - INSERT_TIME, INSERT_TIME_FIELD); + Map FIELD_NAME_TO_FIELD_MAP = Map.of( + CIKeys.KEY_START, KEY_START_FIELD, + CIKeys.KEY_END, KEY_END_FIELD, + CIKeys.VALUE_TYPE, VALUE_TYPE_FIELD, + CIKeys.VALUE, VALUE_FIELD, + CIKeys.INSERT_TIME, INSERT_TIME_FIELD); } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SearchHelper.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SearchHelper.java index 9b5506871ca..639fcc9ee4f 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SearchHelper.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SearchHelper.java @@ -30,6 +30,7 @@ import stroom.query.language.functions.ValString; import stroom.query.language.functions.ValuesConsumer; import stroom.state.impl.ScyllaDbExpressionUtil; +import stroom.util.shared.string.CIKey; import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; @@ -51,21 +52,21 @@ public class SearchHelper { private final Provider sessionProvider; private final CqlIdentifier table; - private final Map columnMap; - private final String valueTypeFieldName; - private final String valueFieldName; + private final Map fieldNameToColumnMap; + private final CIKey valueTypeFieldName; + private final CIKey valueFieldName; private int valueTypePosition = -1; private int valuePosition = -1; public SearchHelper(final Provider sessionProvider, final CqlIdentifier table, - final Map columnMap, - final String valueTypeFieldName, - final String valueFieldName) { + final Map fieldNameToColumnMap, + final CIKey valueTypeFieldName, + final CIKey valueFieldName) { this.sessionProvider = sessionProvider; this.table = table; - this.columnMap = columnMap; + this.fieldNameToColumnMap = fieldNameToColumnMap; this.valueTypeFieldName = valueTypeFieldName; this.valueFieldName = valueFieldName; } @@ -74,16 +75,22 @@ void search(final ExpressionCriteria criteria, final FieldIndex fieldIndex, final DateTimeSettings dateTimeSettings, final ValuesConsumer consumer) { + final List relations = new ArrayList<>(); - ScyllaDbExpressionUtil.getRelations(columnMap, criteria.getExpression(), relations, dateTimeSettings); - final String[] fieldNames = fieldIndex.getFields(); + ScyllaDbExpressionUtil.getRelations( + fieldNameToColumnMap, + criteria.getExpression(), + relations, + dateTimeSettings); + final List fieldNames = fieldIndex.getFieldsAsCIKeys(); final List columns = new ArrayList<>(); + final int fieldCount = fieldNames.size(); int columnPos = 0; - final ValFunction[] valFunctions = new ValFunction[fieldNames.length]; - for (int i = 0; i < fieldNames.length; i++) { - final String fieldName = fieldNames[i]; - final ScyllaDbColumn column = columnMap.get(fieldName); + final ValFunction[] valFunctions = new ValFunction[fieldCount]; + for (int i = 0; i < fieldCount; i++) { + final CIKey fieldName = fieldNames.get(i); + final ScyllaDbColumn column = fieldNameToColumnMap.get(fieldName); if (column != null) { columns.add(column.cqlIdentifier()); final int pos = columnPos; @@ -110,7 +117,7 @@ void search(final ExpressionCriteria criteria, // Add the value type and record the value type position if it is needed. if (valuePosition != -1 && valueTypePosition == -1) { - columns.add(columnMap.get(valueTypeFieldName).cqlIdentifier()); + columns.add(fieldNameToColumnMap.get(valueTypeFieldName).cqlIdentifier()); valueTypePosition = columnPos; } @@ -124,7 +131,7 @@ void search(final ExpressionCriteria criteria, .allowFiltering() .build(); sessionProvider.get().execute(statement).forEach(row -> { - final Val[] values = new Val[fieldNames.length]; + final Val[] values = new Val[fieldCount]; for (int i = 0; i < values.length; i++) { values[i] = valFunctions[i].apply(row); } @@ -152,6 +159,10 @@ private ValFunction convertRow(final DataType dataType, final int pos) { throw new RuntimeException("Unexpected data type: " + dataType); } + + // -------------------------------------------------------------------------------- + + private interface ValFunction extends Function { } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SessionDao.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SessionDao.java index e0c394881b9..bb5c26b13dc 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SessionDao.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SessionDao.java @@ -26,8 +26,11 @@ import stroom.query.language.functions.ValString; import stroom.query.language.functions.ValuesConsumer; import stroom.state.impl.ScyllaDbExpressionUtil; +import stroom.util.NullSafe; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; @@ -49,6 +52,7 @@ import java.util.Objects; import java.util.Optional; import java.util.function.Consumer; +import java.util.function.Function; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.bindMarker; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.deleteFrom; @@ -66,15 +70,11 @@ public class SessionDao extends AbstractStateDao { private static final CqlIdentifier COLUMN_END = CqlIdentifier.fromCql("end"); private static final CqlIdentifier COLUMN_TERMINAL = CqlIdentifier.fromCql("terminal"); - private static final Map COLUMN_MAP = Map.of( - SessionFields.KEY, - new ScyllaDbColumn(SessionFields.KEY, DataTypes.TEXT, COLUMN_KEY), - SessionFields.START, - new ScyllaDbColumn(SessionFields.START, DataTypes.TIMESTAMP, COLUMN_START), - SessionFields.END, - new ScyllaDbColumn(SessionFields.END, DataTypes.TIMESTAMP, COLUMN_END), - SessionFields.TERMINAL, - new ScyllaDbColumn(SessionFields.TERMINAL, DataTypes.BOOLEAN, COLUMN_TERMINAL)); + private static final Map COLUMN_MAP = Map.ofEntries( + StateFieldUtil.createNameToColumnEntry(CIKeys.KEY, DataTypes.TEXT, COLUMN_KEY), + StateFieldUtil.createNameToColumnEntry(CIKeys.START, DataTypes.TIMESTAMP, COLUMN_START), + StateFieldUtil.createNameToColumnEntry(CIKeys.END, DataTypes.TIMESTAMP, COLUMN_END), + StateFieldUtil.createNameToColumnEntry(CIKeys.TERMINAL, DataTypes.BOOLEAN, COLUMN_TERMINAL)); public SessionDao(final Provider sessionProvider, final String tableName) { super(sessionProvider, CqlIdentifier.fromCql(tableName)); @@ -450,28 +450,36 @@ public void flush() { } } + + // -------------------------------------------------------------------------------- + + private static class SessionConsumer implements Consumer { - private final String[] fieldNames; + private final List fieldNames; private final ValuesConsumer consumer; + private static final Map> FIELD_NAME_TO_EXTRACTOR_MAP = Map.of( + CIKeys.KEY, session -> ValString.create(session.key()), + CIKeys.START, session -> ValDate.create(session.start()), + CIKeys.END, session -> ValDate.create(session.end()), + CIKeys.TERMINAL, session -> ValBoolean.create(session.terminal())); + public SessionConsumer(final FieldIndex fieldIndex, final ValuesConsumer consumer) { - this.fieldNames = fieldIndex.getFields(); + this.fieldNames = fieldIndex.getFieldsAsCIKeys(); this.consumer = consumer; } @Override public void accept(final Session session) { - final Val[] values = new Val[fieldNames.length]; - for (int i = 0; i < values.length; i++) { - final String fieldName = fieldNames[i]; - switch (fieldName) { - case SessionFields.KEY -> values[i] = ValString.create(session.key()); - case SessionFields.START -> values[i] = ValDate.create(session.start()); - case SessionFields.END -> values[i] = ValDate.create(session.end()); - case SessionFields.TERMINAL -> values[i] = ValBoolean.create(session.terminal()); - default -> values[i] = ValNull.INSTANCE; - } + final int fieldCount = fieldNames.size(); + final Val[] values = new Val[fieldCount]; + for (int i = 0; i < fieldCount; i++) { + final CIKey fieldName = fieldNames.get(i); + values[i] = NullSafe.getOrElse( + FIELD_NAME_TO_EXTRACTOR_MAP.get(fieldName), + extractor -> extractor.apply(session), + ValNull.INSTANCE); } consumer.accept(Val.of(values)); } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SessionFields.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SessionFields.java index e1ed923bf20..9177ebd41cb 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SessionFields.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/SessionFields.java @@ -1,32 +1,44 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; -import java.util.Arrays; import java.util.List; import java.util.Map; public interface SessionFields { - String KEY = "Key"; - String START = "Start"; - String END = "End"; - String TERMINAL = "Terminal"; - - QueryField KEY_FIELD = QueryField.createText(KEY); - QueryField START_FIELD = QueryField.createDate(START); - QueryField END_FIELD = QueryField.createDate(END); - QueryField TERMINAL_FIELD = QueryField.createText(TERMINAL, false); + QueryField KEY_FIELD = QueryField.createText(CIKeys.KEY, true); + QueryField START_FIELD = QueryField.createDate(CIKeys.START, true); + QueryField END_FIELD = QueryField.createDate(CIKeys.END, true); + QueryField TERMINAL_FIELD = QueryField.createText(CIKeys.TERMINAL, false); - List FIELDS = Arrays.asList( + List FIELDS = List.of( KEY_FIELD, START_FIELD, END_FIELD, TERMINAL_FIELD); - Map FIELD_MAP = Map.of( - KEY, KEY_FIELD, - START, START_FIELD, - END, END_FIELD, - TERMINAL, TERMINAL_FIELD); + Map FIELD_NAME_TO_FIELD_MAP = Map.of( + CIKeys.KEY, KEY_FIELD, + CIKeys.START, START_FIELD, + CIKeys.END, END_FIELD, + CIKeys.TERMINAL, TERMINAL_FIELD); } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateDao.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateDao.java index 5fba821e73d..833bd9bf148 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateDao.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateDao.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.entity.shared.ExpressionCriteria; @@ -6,6 +22,8 @@ import stroom.query.language.functions.ValuesConsumer; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; @@ -40,15 +58,11 @@ public class StateDao extends AbstractStateDao { private static final CqlIdentifier COLUMN_VALUE = CqlIdentifier.fromCql("value"); private static final CqlIdentifier COLUMN_INSERT_TIME = CqlIdentifier.fromCql("insert_time"); - private static final Map COLUMN_MAP = Map.of( - StateFields.KEY, - new ScyllaDbColumn(StateFields.KEY, DataTypes.TEXT, COLUMN_KEY), - StateFields.VALUE_TYPE, - new ScyllaDbColumn(StateFields.VALUE_TYPE, DataTypes.TINYINT, COLUMN_VALUE_TYPE), - StateFields.VALUE, - new ScyllaDbColumn(StateFields.VALUE, DataTypes.BLOB, COLUMN_VALUE), - StateFields.INSERT_TIME, - new ScyllaDbColumn(StateFields.INSERT_TIME, DataTypes.TIMESTAMP, COLUMN_INSERT_TIME)); + private static final Map FIELD_NAME_TO_COLUMN_MAP = Map.ofEntries( + StateFieldUtil.createNameToColumnEntry(CIKeys.KEY, DataTypes.TEXT, COLUMN_KEY), + StateFieldUtil.createNameToColumnEntry(CIKeys.VALUE_TYPE, DataTypes.TINYINT, COLUMN_VALUE_TYPE), + StateFieldUtil.createNameToColumnEntry(CIKeys.VALUE, DataTypes.BLOB, COLUMN_VALUE), + StateFieldUtil.createNameToColumnEntry(CIKeys.INSERT_TIME, DataTypes.TIMESTAMP, COLUMN_INSERT_TIME)); public StateDao(final Provider sessionProvider, final String tableName) { super(sessionProvider, CqlIdentifier.fromCql(tableName)); @@ -128,9 +142,9 @@ public void search(final ExpressionCriteria criteria, final SearchHelper searchHelper = new SearchHelper( sessionProvider, table, - COLUMN_MAP, - StateFields.VALUE_TYPE, - StateFields.VALUE); + FIELD_NAME_TO_COLUMN_MAP, + CIKeys.VALUE_TYPE, + CIKeys.VALUE); searchHelper.search(criteria, fieldIndex, dateTimeSettings, consumer); } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateFieldUtil.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateFieldUtil.java index 784e3963c31..b702862a20d 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateFieldUtil.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateFieldUtil.java @@ -1,10 +1,31 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.datasource.api.v2.QueryField; import stroom.state.shared.StateType; +import stroom.util.shared.string.CIKey; + +import com.datastax.oss.driver.api.core.CqlIdentifier; +import com.datastax.oss.driver.api.core.type.DataType; import java.util.List; import java.util.Map; +import java.util.Map.Entry; public class StateFieldUtil { @@ -18,23 +39,32 @@ public static List getQueryableFields(final StateType stateType) { }; } - public static Map getFieldMap(final StateType stateType) { + public static Map getFieldMap(final StateType stateType) { return switch (stateType) { - case STATE -> StateFields.FIELD_MAP; - case TEMPORAL_STATE -> TemporalStateFields.FIELD_MAP; - case RANGED_STATE -> RangedStateFields.FIELD_MAP; - case TEMPORAL_RANGED_STATE -> TemporalRangedStateFields.FIELD_MAP; - case SESSION -> SessionFields.FIELD_MAP; + case STATE -> StateFields.FIELD_NAME_TO_FIELD_MAP; + case TEMPORAL_STATE -> TemporalStateFields.FIELD_NAME_TO_FIELD_MAP; + case RANGED_STATE -> RangedStateFields.FIELD_NAME_TO_FIELD_MAP; + case TEMPORAL_RANGED_STATE -> TemporalRangedStateFields.FIELD_NAME_TO_FIELD_MAP; + case SESSION -> SessionFields.FIELD_NAME_TO_FIELD_MAP; }; } public static QueryField getTimeField(final StateType stateType) { return switch (stateType) { - case STATE -> null; + case STATE, RANGED_STATE -> null; case TEMPORAL_STATE -> TemporalStateFields.EFFECTIVE_TIME_FIELD; - case RANGED_STATE -> null; case TEMPORAL_RANGED_STATE -> TemporalRangedStateFields.EFFECTIVE_TIME_FIELD; case SESSION -> SessionFields.START_FIELD; }; } + + public static Entry createNameToColumnEntry( + final CIKey name, + final DataType dataType, + final CqlIdentifier identifier) { + + return Map.entry( + name, + new ScyllaDbColumn(name.get(), dataType, identifier)); + } } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateFields.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateFields.java index bb56b295fef..00d0d6ea22e 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateFields.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/StateFields.java @@ -1,6 +1,24 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import java.util.Arrays; import java.util.List; @@ -8,15 +26,10 @@ public interface StateFields { - String KEY = "Key"; - String VALUE_TYPE = "ValueType"; - String VALUE = "Value"; - String INSERT_TIME = "InsertTime"; - - QueryField KEY_FIELD = QueryField.createText(KEY); - QueryField VALUE_TYPE_FIELD = QueryField.createText(VALUE_TYPE, false); - QueryField VALUE_FIELD = QueryField.createText(VALUE, false); - QueryField INSERT_TIME_FIELD = QueryField.createText(INSERT_TIME, false); + QueryField KEY_FIELD = QueryField.createText(CIKeys.KEY, true); + QueryField VALUE_TYPE_FIELD = QueryField.createText(CIKeys.VALUE_TYPE, false); + QueryField VALUE_FIELD = QueryField.createText(CIKeys.VALUE, false); + QueryField INSERT_TIME_FIELD = QueryField.createText(CIKeys.INSERT_TIME, false); List FIELDS = Arrays.asList( KEY_FIELD, @@ -24,9 +37,9 @@ public interface StateFields { VALUE_FIELD, INSERT_TIME_FIELD); - Map FIELD_MAP = Map.of( - KEY, KEY_FIELD, - VALUE_TYPE, VALUE_TYPE_FIELD, - VALUE, VALUE_FIELD, - INSERT_TIME, INSERT_TIME_FIELD); + Map FIELD_NAME_TO_FIELD_MAP = Map.of( + CIKeys.KEY, KEY_FIELD, + CIKeys.VALUE_TYPE, VALUE_TYPE_FIELD, + CIKeys.VALUE, VALUE_FIELD, + CIKeys.INSERT_TIME, INSERT_TIME_FIELD); } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalRangedStateDao.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalRangedStateDao.java index f5101bb4886..9377ec41e19 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalRangedStateDao.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalRangedStateDao.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.entity.shared.ExpressionCriteria; @@ -6,6 +22,8 @@ import stroom.query.language.functions.ValuesConsumer; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; @@ -44,17 +62,14 @@ public class TemporalRangedStateDao extends AbstractStateDao COLUMN_MAP = Map.of( - TemporalRangedStateFields.KEY_START, - new ScyllaDbColumn(TemporalRangedStateFields.KEY_START, DataTypes.BIGINT, COLUMN_KEY_START), - TemporalRangedStateFields.KEY_END, - new ScyllaDbColumn(TemporalRangedStateFields.KEY_END, DataTypes.BIGINT, COLUMN_KEY_END), - TemporalRangedStateFields.EFFECTIVE_TIME, - new ScyllaDbColumn(TemporalRangedStateFields.EFFECTIVE_TIME, DataTypes.TIMESTAMP, COLUMN_EFFECTIVE_TIME), - TemporalRangedStateFields.VALUE_TYPE, - new ScyllaDbColumn(TemporalRangedStateFields.VALUE_TYPE, DataTypes.TINYINT, COLUMN_VALUE_TYPE), - TemporalRangedStateFields.VALUE, - new ScyllaDbColumn(TemporalRangedStateFields.VALUE, DataTypes.BLOB, COLUMN_VALUE)); + + private static final Map NAME_TO_COLUMN_MAP = Map.ofEntries( + StateFieldUtil.createNameToColumnEntry(CIKeys.KEY_START, DataTypes.BIGINT, COLUMN_KEY_START), + StateFieldUtil.createNameToColumnEntry(CIKeys.KEY_END, DataTypes.BIGINT, COLUMN_KEY_END), + StateFieldUtil.createNameToColumnEntry( + CIKeys.EFFECTIVE_TIME, DataTypes.TIMESTAMP, COLUMN_EFFECTIVE_TIME), + StateFieldUtil.createNameToColumnEntry(CIKeys.VALUE_TYPE, DataTypes.TINYINT, COLUMN_VALUE_TYPE), + StateFieldUtil.createNameToColumnEntry(CIKeys.VALUE, DataTypes.BLOB, COLUMN_VALUE)); public TemporalRangedStateDao(final Provider sessionProvider, final String tableName) { super(sessionProvider, CqlIdentifier.fromCql(tableName)); @@ -134,9 +149,9 @@ public void search(final ExpressionCriteria criteria, final SearchHelper searchHelper = new SearchHelper( sessionProvider, table, - COLUMN_MAP, - TemporalRangedStateFields.VALUE_TYPE, - TemporalRangedStateFields.VALUE); + NAME_TO_COLUMN_MAP, + CIKeys.VALUE_TYPE, + CIKeys.VALUE); searchHelper.search(criteria, fieldIndex, dateTimeSettings, consumer); } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalRangedStateFields.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalRangedStateFields.java index 9e5051eb45c..1bd00be7cf1 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalRangedStateFields.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalRangedStateFields.java @@ -1,36 +1,47 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; -import java.util.Arrays; import java.util.List; import java.util.Map; public interface TemporalRangedStateFields { - String KEY_START = "KeyStart"; - String KEY_END = "KeyEnd"; - String EFFECTIVE_TIME = "EffectiveTime"; - String VALUE_TYPE = "ValueType"; - String VALUE = "Value"; - - QueryField KEY_START_FIELD = QueryField.createLong(KEY_START); - QueryField KEY_END_FIELD = QueryField.createText(KEY_END); - QueryField EFFECTIVE_TIME_FIELD = QueryField.createDate(EFFECTIVE_TIME); - QueryField VALUE_TYPE_FIELD = QueryField.createText(VALUE_TYPE, false); - QueryField VALUE_FIELD = QueryField.createText(VALUE, false); + QueryField KEY_START_FIELD = QueryField.createLong(CIKeys.KEY_START, true); + QueryField KEY_END_FIELD = QueryField.createText(CIKeys.KEY_END, true); + QueryField EFFECTIVE_TIME_FIELD = QueryField.createDate(CIKeys.EFFECTIVE_TIME, true); + QueryField VALUE_TYPE_FIELD = QueryField.createText(CIKeys.VALUE_TYPE, false); + QueryField VALUE_FIELD = QueryField.createText(CIKeys.VALUE, false); - List FIELDS = Arrays.asList( + List FIELDS = List.of( KEY_START_FIELD, KEY_END_FIELD, EFFECTIVE_TIME_FIELD, VALUE_TYPE_FIELD, VALUE_FIELD); - Map FIELD_MAP = Map.of( - KEY_START, KEY_START_FIELD, - KEY_END, KEY_END_FIELD, - EFFECTIVE_TIME, EFFECTIVE_TIME_FIELD, - VALUE_TYPE, VALUE_TYPE_FIELD, - VALUE, VALUE_FIELD); + Map FIELD_NAME_TO_FIELD_MAP = Map.of( + CIKeys.KEY_START, KEY_START_FIELD, + CIKeys.KEY_END, KEY_END_FIELD, + CIKeys.EFFECTIVE_TIME, EFFECTIVE_TIME_FIELD, + CIKeys.VALUE_TYPE, VALUE_TYPE_FIELD, + CIKeys.VALUE, VALUE_FIELD); } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalState.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalState.java index 5158a5a543e..677cea39513 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalState.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalState.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.pipeline.refdata.store.FastInfosetUtil; @@ -22,6 +38,10 @@ public String getValueAsString() { }; } + + // -------------------------------------------------------------------------------- + + public static class Builder { private String key; diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalStateDao.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalStateDao.java index cd7af506528..056140e229a 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalStateDao.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalStateDao.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.entity.shared.ExpressionCriteria; @@ -6,6 +22,8 @@ import stroom.query.language.functions.ValuesConsumer; import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; @@ -45,17 +63,12 @@ public class TemporalStateDao extends AbstractStateDao { private static final CqlIdentifier COLUMN_VALUE_TYPE = CqlIdentifier.fromCql("value_type"); private static final CqlIdentifier COLUMN_VALUE = CqlIdentifier.fromCql("value"); - private static final Map COLUMN_MAP = Map.of( - TemporalStateFields.KEY, - new ScyllaDbColumn(TemporalStateFields.KEY, DataTypes.TEXT, COLUMN_KEY), - TemporalStateFields.EFFECTIVE_TIME, - new ScyllaDbColumn(TemporalStateFields.EFFECTIVE_TIME, DataTypes.TIMESTAMP, COLUMN_EFFECTIVE_TIME), - TemporalStateFields.VALUE_TYPE, - new ScyllaDbColumn(TemporalStateFields.VALUE_TYPE, DataTypes.TINYINT, COLUMN_VALUE_TYPE), - TemporalStateFields.VALUE, - new ScyllaDbColumn(TemporalStateFields.VALUE, DataTypes.BLOB, COLUMN_VALUE)); - - + private static final Map FIELD_NAME_TO_COLUMN_MAP = Map.ofEntries( + StateFieldUtil.createNameToColumnEntry(CIKeys.KEY, DataTypes.TEXT, COLUMN_KEY), + StateFieldUtil.createNameToColumnEntry( + CIKeys.EFFECTIVE_TIME, DataTypes.TIMESTAMP, COLUMN_EFFECTIVE_TIME), + StateFieldUtil.createNameToColumnEntry(CIKeys.VALUE_TYPE, DataTypes.TINYINT, COLUMN_VALUE_TYPE), + StateFieldUtil.createNameToColumnEntry(CIKeys.VALUE, DataTypes.BLOB, COLUMN_VALUE)); public TemporalStateDao(final Provider sessionProvider, final String tableName) { super(sessionProvider, CqlIdentifier.fromCql(tableName)); @@ -141,9 +154,9 @@ public void search(final ExpressionCriteria criteria, final SearchHelper searchHelper = new SearchHelper( sessionProvider, table, - COLUMN_MAP, - TemporalStateFields.VALUE_TYPE, - TemporalStateFields.VALUE); + FIELD_NAME_TO_COLUMN_MAP, + CIKeys.VALUE_TYPE, + CIKeys.VALUE); searchHelper.search(criteria, fieldIndex, dateTimeSettings, consumer); } diff --git a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalStateFields.java b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalStateFields.java index caa37025cdb..84032a1b85b 100644 --- a/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalStateFields.java +++ b/stroom-state/stroom-state-impl/src/main/java/stroom/state/impl/dao/TemporalStateFields.java @@ -1,31 +1,44 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.state.impl.dao; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; -import java.util.Arrays; import java.util.List; import java.util.Map; public interface TemporalStateFields { - String KEY = "Key"; - String EFFECTIVE_TIME = "EffectiveTime"; - String VALUE_TYPE = "ValueType"; - String VALUE = "Value"; - QueryField KEY_FIELD = QueryField.createText(KEY); - QueryField EFFECTIVE_TIME_FIELD = QueryField.createDate(EFFECTIVE_TIME); - QueryField VALUE_TYPE_FIELD = QueryField.createText(VALUE_TYPE, false); - QueryField VALUE_FIELD = QueryField.createText(VALUE, false); + QueryField KEY_FIELD = QueryField.createText(CIKeys.KEY, true); + QueryField EFFECTIVE_TIME_FIELD = QueryField.createDate(CIKeys.EFFECTIVE_TIME, true); + QueryField VALUE_TYPE_FIELD = QueryField.createText(CIKeys.VALUE_TYPE, false); + QueryField VALUE_FIELD = QueryField.createText(CIKeys.VALUE, false); - List FIELDS = Arrays.asList( + List FIELDS = List.of( KEY_FIELD, EFFECTIVE_TIME_FIELD, VALUE_TYPE_FIELD, VALUE_FIELD); - Map FIELD_MAP = Map.of( - KEY, KEY_FIELD, - EFFECTIVE_TIME, EFFECTIVE_TIME_FIELD, - VALUE_TYPE, VALUE_TYPE_FIELD, - VALUE, VALUE_FIELD); + Map FIELD_NAME_TO_FIELD_MAP = Map.of( + CIKeys.KEY, KEY_FIELD, + CIKeys.EFFECTIVE_TIME, EFFECTIVE_TIME_FIELD, + CIKeys.VALUE_TYPE, VALUE_TYPE_FIELD, + CIKeys.VALUE, VALUE_FIELD); } diff --git a/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestRangedStateDao.java b/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestRangedStateDao.java similarity index 87% rename from stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestRangedStateDao.java rename to stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestRangedStateDao.java index 49473d25f6c..e7d1c19d692 100644 --- a/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestRangedStateDao.java +++ b/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestRangedStateDao.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,20 +12,16 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package stroom.state.impl; +package stroom.state.impl.dao; import stroom.entity.shared.ExpressionCriteria; import stroom.pipeline.refdata.store.StringValue; import stroom.query.api.v2.ExpressionOperator; import stroom.query.language.functions.FieldIndex; -import stroom.state.impl.dao.RangedState; -import stroom.state.impl.dao.RangedStateDao; -import stroom.state.impl.dao.RangedStateFields; -import stroom.state.impl.dao.RangedStateRequest; -import stroom.state.impl.dao.State; +import stroom.state.impl.ScyllaDbUtil; +import stroom.util.shared.string.CIKeys; import org.junit.jupiter.api.Test; @@ -57,9 +53,12 @@ void testDao() { assertThat(res.getValueAsString()).isEqualTo("test99"); final FieldIndex fieldIndex = new FieldIndex(); - fieldIndex.create(RangedStateFields.KEY_START); + fieldIndex.create(CIKeys.KEY_START); final AtomicInteger count = new AtomicInteger(); - rangedStateDao.search(new ExpressionCriteria(ExpressionOperator.builder().build()), fieldIndex, null, + rangedStateDao.search( + new ExpressionCriteria(ExpressionOperator.builder().build()), + fieldIndex, + null, v -> count.incrementAndGet()); assertThat(count.get()).isEqualTo(1); }); diff --git a/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestSessionDao.java b/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestSessionDao.java similarity index 91% rename from stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestSessionDao.java rename to stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestSessionDao.java index 063bf25bca4..ce00c8a8d15 100644 --- a/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestSessionDao.java +++ b/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestSessionDao.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,20 +12,18 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package stroom.state.impl; +package stroom.state.impl.dao; import stroom.entity.shared.ExpressionCriteria; import stroom.query.api.v2.ExpressionOperator; import stroom.query.api.v2.ExpressionTerm.Condition; import stroom.query.language.functions.FieldIndex; import stroom.query.language.functions.ValDate; -import stroom.state.impl.dao.Session; -import stroom.state.impl.dao.SessionDao; -import stroom.state.impl.dao.SessionFields; -import stroom.state.impl.dao.TemporalStateRequest; +import stroom.state.impl.InstantRange; +import stroom.state.impl.ScyllaDbUtil; +import stroom.util.shared.string.CIKeys; import org.junit.jupiter.api.Test; @@ -56,10 +54,10 @@ void testDao() { final AtomicInteger count = new AtomicInteger(); final FieldIndex fieldIndex = new FieldIndex(); - fieldIndex.create(SessionFields.KEY); - fieldIndex.create(SessionFields.START); - fieldIndex.create(SessionFields.END); - fieldIndex.create(SessionFields.TERMINAL); + fieldIndex.create(CIKeys.KEY); + fieldIndex.create(CIKeys.START); + fieldIndex.create(CIKeys.END); + fieldIndex.create(CIKeys.TERMINAL); final ValDate minTime = ValDate.create(outerRange.min()); final ValDate maxTime = ValDate.create(outerRange.max()); diff --git a/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestStateDao.java b/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestStateDao.java similarity index 88% rename from stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestStateDao.java rename to stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestStateDao.java index 4f48ea61038..5f8fe24ecba 100644 --- a/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestStateDao.java +++ b/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestStateDao.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,19 +12,16 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package stroom.state.impl; +package stroom.state.impl.dao; import stroom.entity.shared.ExpressionCriteria; import stroom.pipeline.refdata.store.StringValue; import stroom.query.api.v2.ExpressionOperator; import stroom.query.language.functions.FieldIndex; -import stroom.state.impl.dao.State; -import stroom.state.impl.dao.StateDao; -import stroom.state.impl.dao.StateFields; -import stroom.state.impl.dao.StateRequest; +import stroom.state.impl.ScyllaDbUtil; +import stroom.util.shared.string.CIKeys; import org.junit.jupiter.api.Test; @@ -55,9 +52,12 @@ void testDao() { assertThat(res.getValueAsString()).isEqualTo("test99"); final FieldIndex fieldIndex = new FieldIndex(); - fieldIndex.create(StateFields.KEY); + fieldIndex.create(CIKeys.KEY); final AtomicInteger count = new AtomicInteger(); - stateDao.search(new ExpressionCriteria(ExpressionOperator.builder().build()), fieldIndex, null, + stateDao.search( + new ExpressionCriteria(ExpressionOperator.builder().build()), + fieldIndex, + null, v -> count.incrementAndGet()); assertThat(count.get()).isEqualTo(1); }); diff --git a/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestTemporalRangedStateDao.java b/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestTemporalRangedStateDao.java similarity index 92% rename from stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestTemporalRangedStateDao.java rename to stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestTemporalRangedStateDao.java index 55b308a7ac0..b91f6ed6641 100644 --- a/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestTemporalRangedStateDao.java +++ b/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestTemporalRangedStateDao.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,20 +12,16 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package stroom.state.impl; +package stroom.state.impl.dao; import stroom.entity.shared.ExpressionCriteria; import stroom.pipeline.refdata.store.StringValue; import stroom.query.api.v2.ExpressionOperator; import stroom.query.language.functions.FieldIndex; -import stroom.state.impl.dao.RangedStateFields; -import stroom.state.impl.dao.TemporalRangedState; -import stroom.state.impl.dao.TemporalRangedStateDao; -import stroom.state.impl.dao.TemporalRangedStateRequest; -import stroom.state.impl.dao.TemporalState; +import stroom.state.impl.ScyllaDbUtil; +import stroom.util.shared.string.CIKeys; import org.junit.jupiter.api.Test; @@ -59,7 +55,7 @@ void testDao() { assertThat(res.getValueAsString()).isEqualTo("test"); final FieldIndex fieldIndex = new FieldIndex(); - fieldIndex.create(RangedStateFields.KEY_START); + fieldIndex.create(CIKeys.KEY_START); final AtomicInteger count = new AtomicInteger(); stateDao.search(new ExpressionCriteria(ExpressionOperator.builder().build()), fieldIndex, null, v -> count.incrementAndGet()); diff --git a/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestTemporalStateDao.java b/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestTemporalStateDao.java similarity index 86% rename from stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestTemporalStateDao.java rename to stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestTemporalStateDao.java index 5099cc1d0de..50983d82dd1 100644 --- a/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/TestTemporalStateDao.java +++ b/stroom-state/stroom-state-impl/src/test/java/stroom/state/impl/dao/TestTemporalStateDao.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,19 +12,16 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package stroom.state.impl; +package stroom.state.impl.dao; import stroom.entity.shared.ExpressionCriteria; import stroom.pipeline.refdata.store.StringValue; import stroom.query.api.v2.ExpressionOperator; import stroom.query.language.functions.FieldIndex; -import stroom.state.impl.dao.TemporalState; -import stroom.state.impl.dao.TemporalStateDao; -import stroom.state.impl.dao.TemporalStateFields; -import stroom.state.impl.dao.TemporalStateRequest; +import stroom.state.impl.ScyllaDbUtil; +import stroom.util.shared.string.CIKeys; import org.junit.jupiter.api.Test; @@ -58,9 +55,12 @@ void testDao() { assertThat(res.getValueAsString()).isEqualTo("test"); final FieldIndex fieldIndex = new FieldIndex(); - fieldIndex.create(TemporalStateFields.KEY); + fieldIndex.create(CIKeys.KEY); final AtomicInteger count = new AtomicInteger(); - stateDao.search(new ExpressionCriteria(ExpressionOperator.builder().build()), fieldIndex, null, + stateDao.search(new ExpressionCriteria( + ExpressionOperator.builder().build()), + fieldIndex, + null, v -> count.incrementAndGet()); assertThat(count.get()).isEqualTo(100); }); @@ -105,10 +105,10 @@ void testCondense() { } private void insertData(final TemporalStateDao stateDao, - final Instant refTime, - final String value, - final int rows, - final long deltaSeconds) { + final Instant refTime, + final String value, + final int rows, + final long deltaSeconds) { final ByteBuffer byteBuffer = ByteBuffer.wrap((value).getBytes(StandardCharsets.UTF_8)); for (int i = 0; i < rows; i++) { final Instant effectiveTime = refTime.plusSeconds(i * deltaSeconds); diff --git a/stroom-statistics/stroom-statistics-impl-hbase/src/main/java/stroom/statistics/impl/hbase/entity/StroomStatsStoreStoreImpl.java b/stroom-statistics/stroom-statistics-impl-hbase/src/main/java/stroom/statistics/impl/hbase/entity/StroomStatsStoreStoreImpl.java index 70a31bd7a79..5ac24004dc1 100644 --- a/stroom-statistics/stroom-statistics-impl-hbase/src/main/java/stroom/statistics/impl/hbase/entity/StroomStatsStoreStoreImpl.java +++ b/stroom-statistics/stroom-statistics-impl-hbase/src/main/java/stroom/statistics/impl/hbase/entity/StroomStatsStoreStoreImpl.java @@ -189,8 +189,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/entity/StatisticStoreStoreImpl.java b/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/entity/StatisticStoreStoreImpl.java index f5499db410b..1fbdb1833da 100644 --- a/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/entity/StatisticStoreStoreImpl.java +++ b/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/entity/StatisticStoreStoreImpl.java @@ -196,8 +196,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/stroom-statistics/stroom-statistics-impl-sql/src/test/java/stroom/statistics/impl/sql/TestSQLStatisticEventStore2.java b/stroom-statistics/stroom-statistics-impl-sql/src/test/java/stroom/statistics/impl/sql/TestSQLStatisticEventStore2.java index 9629a82afb9..ca8ce52d1f5 100644 --- a/stroom-statistics/stroom-statistics-impl-sql/src/test/java/stroom/statistics/impl/sql/TestSQLStatisticEventStore2.java +++ b/stroom-statistics/stroom-statistics-impl-sql/src/test/java/stroom/statistics/impl/sql/TestSQLStatisticEventStore2.java @@ -1,5 +1,5 @@ /* - * Copyright 2017 Crown Copyright + * Copyright 2017-2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ import stroom.util.date.DateUtil; import jakarta.ws.rs.BadRequestException; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -326,6 +327,8 @@ void testBuildCriteria_invalidDateTermOnlyOneDate() { @Test void testBuildCriteria_validDateTermOtherTermMissingFieldName() { + Assertions.setMaxStackTraceElementsDisplayed(20); + assertThatThrownBy(() -> { final ExpressionOperator.Builder rootOperator = ExpressionOperator.builder(); @@ -343,7 +346,8 @@ void testBuildCriteria_validDateTermOtherTermMissingFieldName() { dataSource.setName("MyDataSource"); StatStoreCriteriaBuilder.buildCriteria(dataSource, rootOperator.build(), null); - }).isInstanceOf(BadRequestException.class); + }) + .isInstanceOf(BadRequestException.class); } @Test diff --git a/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableDual.java b/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableDual.java index 2c9e8576bbf..11ac1c6f1b4 100644 --- a/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableDual.java +++ b/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableDual.java @@ -26,7 +26,9 @@ import stroom.query.language.functions.ValString; import stroom.query.language.functions.ValuesConsumer; import stroom.searchable.api.Searchable; +import stroom.util.NullSafe; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import java.util.Collections; import java.util.List; @@ -39,11 +41,12 @@ public class SearchableDual implements Searchable { "Dual", "Dual"); - private static final QueryField DUMMY_FIELD = QueryField.createText( - "Dummy", true); + private static final QueryField DUMMY_FIELD = QueryField.createText(CIKey.ofStaticKey("Dummy"), true); private static final List FIELDS = Collections.singletonList(DUMMY_FIELD); + private static final ValString DUMMY_VALUE = ValString.create("X"); + @Override public DocRef getDocRef() { return DOC_REF; @@ -76,11 +79,10 @@ public QueryField getTimeField() { public void search(final ExpressionCriteria criteria, final FieldIndex fieldIndex, final ValuesConsumer consumer) { - final String[] fields = fieldIndex.getFields(); - final Val[] valArr = new Val[fields.length]; - for (int i = 0; i < fields.length; i++) { - valArr[i] = ValString.create("X"); - } + final List fields = fieldIndex.getFieldsAsCIKeys(); + final Val[] valArr = NullSafe.stream(fields) + .map(fieldName -> DUMMY_VALUE) + .toArray(Val[]::new); consumer.accept(Val.of(valArr)); } } diff --git a/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableRows.java b/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableRows.java new file mode 100644 index 00000000000..d84af13071e --- /dev/null +++ b/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableRows.java @@ -0,0 +1,220 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.task.impl; + +import stroom.datasource.api.v2.ConditionSet; +import stroom.datasource.api.v2.FieldType; +import stroom.datasource.api.v2.FindFieldCriteria; +import stroom.datasource.api.v2.QueryField; +import stroom.docref.DocRef; +import stroom.entity.shared.ExpressionCriteria; +import stroom.query.api.v2.ExpressionItem; +import stroom.query.api.v2.ExpressionOperator; +import stroom.query.api.v2.ExpressionTerm; +import stroom.query.api.v2.ExpressionTerm.Condition; +import stroom.query.common.v2.FieldInfoResultPageBuilder; +import stroom.query.common.v2.Sizes; +import stroom.query.common.v2.SizesProvider; +import stroom.query.language.functions.FieldIndex; +import stroom.query.language.functions.Val; +import stroom.query.language.functions.ValInteger; +import stroom.query.language.functions.ValuesConsumer; +import stroom.searchable.api.Searchable; +import stroom.util.NullSafe; +import stroom.util.PredicateUtil; +import stroom.util.shared.PageRequest; +import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; + +import jakarta.inject.Inject; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; + +public class SearchableRows implements Searchable { + + private static final int MAX_ROWS = 100_000; + + private static final DocRef DOC_REF = new DocRef( + "Searchable", + "Rows", + "Rows"); + + private static final QueryField ROW_NUM_FIELD = QueryField.builder() + .fldName(CIKey.ofStaticKey("RowNum")) + .fldType(FieldType.INTEGER) + .conditionSet(ConditionSet.DEFAULT_NUMERIC) + .queryable(true) + .build(); + + private static final List FIELDS = Collections.singletonList(ROW_NUM_FIELD); + + private static final ExpressionOperator EMPTY_OPERATOR = ExpressionOperator.builder().build(); + private static final Predicate TRUE_PREDICATE = val -> true; + private static final Predicate FALSE_PREDICATE = val -> false; + + private final SizesProvider sizesProvider; + + @Inject + public SearchableRows(final SizesProvider sizesProvider) { + this.sizesProvider = sizesProvider; + } + + @Override + public DocRef getDocRef() { + return DOC_REF; + } + + @Override + public ResultPage getFieldInfo(final FindFieldCriteria criteria) { + if (!DOC_REF.equals(criteria.getDataSourceRef())) { + return ResultPage.empty(); + } + return FieldInfoResultPageBuilder.builder(criteria) + .addAll(FIELDS) + .build(); + } + + @Override + public int getFieldCount(final DocRef docRef) { + return 1; + } + + @Override + public Optional fetchDocumentation(final DocRef docRef) { + return Optional.empty(); + } + + @Override + public QueryField getTimeField() { + return null; + } + + @Override + public void search(final ExpressionCriteria criteria, + final FieldIndex fieldIndex, + final ValuesConsumer consumer) { + final List fields = fieldIndex.getFieldsAsCIKeys(); + final PageRequest pageRequest = criteria.getPageRequest(); + final int offset = NullSafe.getOrElse(pageRequest, PageRequest::getOffset, 0); + final int pageLen = NullSafe.getOrElse(pageRequest, PageRequest::getLength, MAX_ROWS); + final int maxOffsetExc = offset + pageLen; + final CIKey rowNumKey = ROW_NUM_FIELD.getFldNameAsCIKey(); + final int rowNumFieldIdx = fields.indexOf(rowNumKey); + final Sizes sizes = sizesProvider.getDefaultMaxResultsSizes(); + final long maxRows = Math.min(sizes.size(0), MAX_ROWS); + + final Predicate predicate = createPredicate(criteria); + + // TODO fix sorting + int rowNum; + for (int i = offset; i < maxOffsetExc; i++) { + // Convert the values into one based + rowNum = i + 1; + if (rowNum > maxRows) { + break; + } + if (predicate == null || predicate.test(rowNum)) { + final Val[] valArr = new Val[fields.size()]; + valArr[rowNumFieldIdx] = ValInteger.create(rowNum); + consumer.accept(valArr); + } + } + } + + private Predicate createPredicate(final ExpressionCriteria criteria) { + final ExpressionOperator expression = criteria.getExpression(); + if (expression == null || EMPTY_OPERATOR.equals(expression)) { + return null; + } else { + if (expression.enabled()) { + return createPredicate(criteria.getExpression()); + } else { + return null; + } + } + } + + private Predicate createPredicate(final ExpressionOperator expressionOperator) { + final List children = expressionOperator.getChildren(); + final List> childPredicates = new ArrayList<>(children.size()); + for (final ExpressionItem expressionItem : children) { + if (expressionItem != null && expressionItem.enabled()) { + final Predicate childPredicate; + if (expressionItem instanceof ExpressionOperator childOperator) { + childPredicate = createPredicate(childOperator); + } else if (expressionItem instanceof ExpressionTerm childTerm) { + childPredicate = createPredicate(childTerm); + } else { + throw new IllegalArgumentException("Unknown type " + expressionItem.getClass()); + } + NullSafe.consume(childPredicate, childPredicates::add); + } + } + if (children.isEmpty()) { + return null; + } else { + return switch (expressionOperator.op()) { + case OR -> PredicateUtil.orPredicates(childPredicates, null); + case AND -> PredicateUtil.andPredicates(childPredicates, null); + case NOT -> { + final Predicate predicate = PredicateUtil.andPredicates(childPredicates, null); + if (predicate != null) { + yield Predicate.not(predicate); + } else { + yield null; + } + } + }; + } + } + + private Predicate createPredicate(final ExpressionTerm expressionTerm) { + if (expressionTerm == null || !expressionTerm.enabled()) { + return null; + } else { + final String value = expressionTerm.getValue(); + Objects.requireNonNull(value); + final Condition condition = expressionTerm.getCondition(); + return switch (condition) { + case EQUALS -> i -> i == valToInt(value); + case NOT_EQUALS -> i -> i != valToInt(value); + case BETWEEN -> { + final String[] values = value.split(","); + final int[] numbers = new int[values.length]; + for (int j = 0; j < values.length; j++) { + numbers[j] = Integer.parseInt(values[j].trim()); + } + yield i -> i >= numbers[0] && i <= numbers[1]; + } + case GREATER_THAN -> i -> i > valToInt(value); + case GREATER_THAN_OR_EQUAL_TO -> i -> i >= valToInt(value); + case LESS_THAN -> i -> i < valToInt(value); + case LESS_THAN_OR_EQUAL_TO -> i -> i <= valToInt(value); + default -> throw new UnsupportedOperationException("Condition " + condition + " not supported"); + }; + } + } + + private int valToInt(final String val) { + return Integer.parseInt(Objects.requireNonNull(val)); + } +} diff --git a/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableTaskProgress.java b/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableTaskProgress.java index 3f5e8f83ee8..7668982eddb 100644 --- a/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableTaskProgress.java +++ b/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/SearchableTaskProgress.java @@ -44,6 +44,7 @@ import stroom.task.shared.TaskResource; import stroom.util.NullSafe; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import jakarta.inject.Inject; import org.slf4j.Logger; @@ -140,38 +141,42 @@ public void search(final ExpressionCriteria criteria, .map(ResultPage::getValues) .flatMap(List::stream) .map(taskProgress -> { - final Map attributeMap = new HashMap<>(); - attributeMap.put(TaskManagerFields.FIELD_NODE, taskProgress.getNodeName()); - attributeMap.put(TaskManagerFields.FIELD_NAME, taskProgress.getTaskName()); - attributeMap.put(TaskManagerFields.FIELD_USER, taskProgress.getUserName()); - attributeMap.put(TaskManagerFields.FIELD_SUBMIT_TIME, taskProgress.getSubmitTimeMs()); - attributeMap.put(TaskManagerFields.FIELD_AGE, taskProgress.getAgeMs()); - attributeMap.put(TaskManagerFields.FIELD_INFO, taskProgress.getTaskInfo()); + final Map attributeMap = new HashMap<>(); + attributeMap.put(TaskManagerFields.NODE.getFldNameAsCIKey(), taskProgress.getNodeName()); + attributeMap.put(TaskManagerFields.NAME.getFldNameAsCIKey(), taskProgress.getTaskName()); + attributeMap.put(TaskManagerFields.USER.getFldNameAsCIKey(), taskProgress.getUserName()); + attributeMap.put( + TaskManagerFields.SUBMIT_TIME.getFldNameAsCIKey(), taskProgress.getSubmitTimeMs()); + attributeMap.put(TaskManagerFields.AGE.getFldNameAsCIKey(), taskProgress.getAgeMs()); + attributeMap.put(TaskManagerFields.INFO.getFldNameAsCIKey(), taskProgress.getTaskInfo()); return attributeMap; }) .filter(attributeMap -> expressionMatcher.match(attributeMap, criteria.getExpression())) .forEach(attributeMap -> { - final String[] fields = fieldIndex.getFields(); - final Val[] arr = new Val[fields.length]; - for (int i = 0; i < fields.length; i++) { - final String fieldName = fields[i]; + final List fields = fieldIndex.getFields(); + final int fieldCount = fields.size(); + + final Val[] arr = new Val[fieldCount]; + for (int i = 0; i < fieldCount; i++) { + final String fieldName = fields.get(i); Val val = ValNull.INSTANCE; if (fieldName != null) { - final Object o = attributeMap.get(fieldName); + final Object o = attributeMap.get(TaskManagerFields.createCIKey(fieldName)); if (o != null) { - if (o instanceof String) { - val = ValString.create((String) o); - } else if (o instanceof Long) { - final long aLong = (long) o; - if (TaskManagerFields.FIELD_SUBMIT_TIME.equals(fieldName)) { - val = ValDate.create(aLong); - } else if (TaskManagerFields.FIELD_AGE.equals(fieldName)) { - val = ValDuration.create(aLong); - } else { - val = ValLong.create(aLong); + switch (o) { + case String str -> val = ValString.create(str); + case Long aLong -> { + if (TaskManagerFields.FIELD_SUBMIT_TIME.equals(fieldName)) { + val = ValDate.create(aLong); + } else if (TaskManagerFields.FIELD_AGE.equals(fieldName)) { + val = ValDuration.create(aLong); + } else { + val = ValLong.create(aLong); + } + } + case Integer anInt -> val = ValInteger.create(anInt); + default -> { } - } else if (o instanceof Integer) { - val = ValInteger.create((int) o); } } } diff --git a/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/TaskManagerFields.java b/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/TaskManagerFields.java index 8e27baab49f..12d1ae83b2f 100644 --- a/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/TaskManagerFields.java +++ b/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/TaskManagerFields.java @@ -1,6 +1,23 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.task.impl; import stroom.datasource.api.v2.QueryField; +import stroom.util.shared.string.CIKey; import java.util.ArrayList; import java.util.List; @@ -17,33 +34,40 @@ public class TaskManagerFields { public static final String FIELD_AGE = "Age"; public static final String FIELD_INFO = "Info"; - private static final List FIELDS = new ArrayList<>(); - private static final Map FIELD_MAP; - - public static final QueryField NODE = QueryField.createText(FIELD_NODE); - public static final QueryField NAME = QueryField.createText(FIELD_NAME); - public static final QueryField USER = QueryField.createText(FIELD_USER); - public static final QueryField SUBMIT_TIME = QueryField.createDate(FIELD_SUBMIT_TIME); - public static final QueryField AGE = QueryField.createLong(FIELD_AGE); - public static final QueryField INFO = QueryField.createText(FIELD_INFO); - - static { - FIELDS.add(NODE); - FIELDS.add(NAME); - FIELDS.add(USER); - FIELDS.add(SUBMIT_TIME); - FIELDS.add(AGE); - FIELDS.add(INFO); - - FIELD_MAP = FIELDS.stream() - .collect(Collectors.toMap(QueryField::getFldName, Function.identity())); - } + public static final QueryField NODE = QueryField.createText(CIKey.ofStaticKey(FIELD_NODE), true); + public static final QueryField NAME = QueryField.createText(CIKey.ofStaticKey(FIELD_NAME), true); + public static final QueryField USER = QueryField.createText(CIKey.ofStaticKey(FIELD_USER), true); + public static final QueryField SUBMIT_TIME = QueryField.createDate( + CIKey.ofStaticKey(FIELD_SUBMIT_TIME), true); + public static final QueryField AGE = QueryField.createLong(CIKey.ofStaticKey(FIELD_AGE), true); + public static final QueryField INFO = QueryField.createText(CIKey.ofStaticKey(FIELD_INFO), true); + + private static final List FIELDS = List.of( + NODE, + NAME, + USER, + SUBMIT_TIME, + AGE, + INFO); + + private static final Map FIELD_MAP = FIELDS.stream() + .collect(Collectors.toMap( + QueryField::getFldNameAsCIKey, + Function.identity())); + + private static final Map NAME_TO_KEY_MAP = FIELD_MAP.keySet() + .stream() + .collect(Collectors.toMap(CIKey::get, Function.identity())); public static List getFields() { return new ArrayList<>(FIELDS); } - public static Map getFieldMap() { + public static Map getFieldMap() { return FIELD_MAP; } + + public static CIKey createCIKey(final String fieldName) { + return CIKey.of(fieldName, NAME_TO_KEY_MAP); + } } diff --git a/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/TaskModule.java b/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/TaskModule.java index 82f2c9760df..e63b2c7bfaf 100644 --- a/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/TaskModule.java +++ b/stroom-task/stroom-task-impl/src/main/java/stroom/task/impl/TaskModule.java @@ -51,7 +51,8 @@ protected void configure() { // TODO 10/03/2022 AT: Need to move SearchableDual to a better home GuiceUtil.buildMultiBinder(binder(), Searchable.class) .addBinding(SearchableTaskProgress.class) - .addBinding(SearchableDual.class); + .addBinding(SearchableDual.class) + .addBinding(SearchableRows.class); // Make sure the first thing to start and the last thing to stop is the task manager. LifecycleBinder.create(binder()) diff --git a/stroom-util-shared/build.gradle b/stroom-util-shared/build.gradle index 77bc79b69aa..06e87ac49a7 100644 --- a/stroom-util-shared/build.gradle +++ b/stroom-util-shared/build.gradle @@ -9,12 +9,15 @@ dependencies { implementation libs.swagger_annotations testImplementation project(':stroom-test-common') + testImplementation project(':stroom-util') testImplementation libs.assertj_core testImplementation libs.commons_lang testImplementation libs.guice testImplementation libs.jackson_core testImplementation libs.jackson_databind + testImplementation libs.jmh_core + testImplementation libs.jmh_generator_annprocess testImplementation libs.junit_jupiter_api testImplementation libs.junit_jupiter_params testImplementation libs.slf4j_api diff --git a/stroom-util-shared/src/main/java/stroom/util/shared/CompareUtil.java b/stroom-util-shared/src/main/java/stroom/util/shared/CompareUtil.java index 765fbc5fadf..d7035275288 100644 --- a/stroom-util-shared/src/main/java/stroom/util/shared/CompareUtil.java +++ b/stroom-util-shared/src/main/java/stroom/util/shared/CompareUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -66,7 +66,7 @@ public static int compareBoolean(final Boolean l1, final Boolean l2) { return l1.compareTo(l2); } - public static int compareString(final String l1, final String l2) { + public static int compareStringIgnoreCase(final String l1, final String l2) { if (l1 == null && l2 == null) { return 0; } @@ -156,9 +156,15 @@ public static Comparator buildCriteriaComparator( return comparator; } + /** + * Case in-sensitive comparator that places any null values first. + * Copes with null values for T and for the String extracted from T. + */ public static Comparator getNullSafeCaseInsensitiveComparator(final Function extractor) { return Comparator.nullsFirst( - Comparator.comparing(extractor, String.CASE_INSENSITIVE_ORDER)); + Comparator.comparing( + extractor, + Comparator.nullsFirst(String.CASE_INSENSITIVE_ORDER))); } /** @@ -236,4 +242,19 @@ public static > Comparator getNullSafe public static Comparator name(final String name, final Comparator comparator) { return new NamedComparator<>(name, comparator); } + + + /** + * Normalises an {@link Comparable#compareTo(Object)} result into -1, 0, or 1. + * Useful for doing equality assertions on comparators. + */ + public static int normalise(final int compareResult) { + if (compareResult < 0) { + return -1; + } else if (compareResult > 0) { + return 1; + } else { + return compareResult; + } + } } diff --git a/stroom-util-shared/src/main/java/stroom/util/shared/GwtNullSafe.java b/stroom-util-shared/src/main/java/stroom/util/shared/GwtNullSafe.java index 6bb2bc0d552..99de7d2a196 100644 --- a/stroom-util-shared/src/main/java/stroom/util/shared/GwtNullSafe.java +++ b/stroom-util-shared/src/main/java/stroom/util/shared/GwtNullSafe.java @@ -277,14 +277,6 @@ public static boolean isNonEmptyString(final String str) { return str != null && !str.isEmpty(); } - public static Optional nonBlank(final String str) { - if (isBlankString(str)) { - return Optional.empty(); - } else { - return Optional.of(str); - } - } - /** * @return str trimmed or an empty string if str is null */ @@ -294,6 +286,14 @@ public static String trim(final String str) { : ""; } + public static Optional nonBlank(final String str) { + if (isBlankString(str)) { + return Optional.empty(); + } else { + return Optional.of(str); + } + } + public static String join(final CharSequence delimiter, final CharSequence... elements) { if (elements == null || elements.length == 0) { return ""; diff --git a/stroom-util-shared/src/main/java/stroom/util/shared/query/FieldNames.java b/stroom-util-shared/src/main/java/stroom/util/shared/query/FieldNames.java new file mode 100644 index 00000000000..486f4125e69 --- /dev/null +++ b/stroom-util-shared/src/main/java/stroom/util/shared/query/FieldNames.java @@ -0,0 +1,98 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.shared.query; + +import stroom.util.shared.string.CIKey; +import stroom.util.shared.string.CIKeys; + +import java.util.Objects; + +public class FieldNames { + + /** + * For field '{@code __time__}' + */ + public static final CIKey DEFAULT_TIME_FIELD_KEY = CIKeys.UNDERSCORE_TIME; + /** + * Field '{@code __time__}' + */ + public static final String DEFAULT_TIME_FIELD_NAME = DEFAULT_TIME_FIELD_KEY.get(); + /** + * For field '{@code EventTime}' + */ + public static final CIKey FALLBACK_TIME_FIELD_KEY = CIKeys.EVENT_TIME; + /** + * Field '{@code EventTime}' + */ + public static final String FALLBACK_TIME_FIELD_NAME = FALLBACK_TIME_FIELD_KEY.get(); + + /** + * For field '{@code __stream_id__}' + */ + public static final CIKey DEFAULT_STREAM_ID_FIELD_KEY = CIKeys.UNDERSCORE_STREAM_ID; + /** + * Field '{@code __stream_id__}' + */ + public static final String DEFAULT_STREAM_ID_FIELD_NAME = DEFAULT_STREAM_ID_FIELD_KEY.get(); + + /** + * For field '{@code StreamId}' + */ + public static final CIKey FALLBACK_STREAM_ID_FIELD_KEY = CIKeys.STREAM_ID; + /** + * Field '{@code StreamId}' + */ + public static final String FALLBACK_STREAM_ID_FIELD_NAME = FALLBACK_STREAM_ID_FIELD_KEY.get(); + + /** + * For field '{@code __event_id__}' + */ + public static final CIKey DEFAULT_EVENT_ID_FIELD_KEY = CIKeys.UNDERSCORE_EVENT_ID; + /** + * Field '{@code __event_id__}' + */ + public static final String DEFAULT_EVENT_ID_FIELD_NAME = DEFAULT_EVENT_ID_FIELD_KEY.get(); + + /** + * For field '{@code EventId}' + */ + public static final CIKey FALLBACK_EVENT_ID_FIELD_KEY = CIKeys.EVENT_ID; + /** + * Field '{@code EventId}' + */ + public static final String FALLBACK_EVENT_ID_FIELD_NAME = FALLBACK_EVENT_ID_FIELD_KEY.get(); + + private FieldNames() { + // Static stuff only + } + + /** + * @return True if fieldName matches the special Stream ID field. + */ + public static boolean isStreamIdFieldName(final String fieldName) { + return Objects.equals(DEFAULT_STREAM_ID_FIELD_NAME, fieldName) + || Objects.equals(FALLBACK_STREAM_ID_FIELD_NAME, fieldName); + } + + /** + * @return True if fieldName matches the special Event ID field. + */ + public static boolean isEventIdFieldName(final String fieldName) { + return Objects.equals(DEFAULT_EVENT_ID_FIELD_NAME, fieldName) + || Objects.equals(FALLBACK_EVENT_ID_FIELD_NAME, fieldName); + } +} diff --git a/stroom-util-shared/src/main/java/stroom/util/shared/string/CIHashMap.java b/stroom-util-shared/src/main/java/stroom/util/shared/string/CIHashMap.java new file mode 100644 index 00000000000..3293dbc3acd --- /dev/null +++ b/stroom-util-shared/src/main/java/stroom/util/shared/string/CIHashMap.java @@ -0,0 +1,298 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.shared.string; + +import stroom.util.shared.GwtNullSafe; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A standard {@link HashMap} keyed with case-insensitive {@link CIKey} keys. + *

+ * Various methods of {@link HashMap} are overridden so you can interact with the + * map using {@link String} or {@link CIKey} keys, with {@link String} keys getting + * converted to {@link CIKey} on the fly. + *

+ */ +public class CIHashMap extends HashMap { + + @SuppressWarnings("rawtypes") + private static final Map EMPTY = Collections.emptyMap(); + + public static CIHashMap empty() { + //noinspection unchecked + return (CIHashMap) EMPTY; + } + + public CIHashMap(final int initialCapacity, final float loadFactor) { + super(initialCapacity, loadFactor); + } + + public CIHashMap(final int initialCapacity) { + super(initialCapacity); + } + + public CIHashMap() { + super(); + } + + public CIHashMap(final Map m) { + super(m); + } + + public V put(final String key, final V value) { + // CIString used to trim all keys + return super.put(CIKey.of(key), value); + } + + /** + * Converts key into a {@link CIKey} before calling {@link CIHashMap#get(Object)} + */ + // Overload all the super methods that take key as an Object as the compiler + // won't spot people using a String key. + public V get(final String key) { + return super.get(CIKey.of(key)); + } + + public V get(final CIKey key) { + return super.get(key); + } + + public boolean replace(final String key, final V oldValue, final V newValue) { + return super.replace(CIKey.of(key), oldValue, newValue); + } + + public boolean replace(final CIKey key, final V oldValue, final V newValue) { + return super.replace(key, oldValue, newValue); + } + + public V replace(final String key, final V value) { + return super.replace(CIKey.of(key), value); + } + + public V replace(final CIKey key, final V value) { + return super.replace(key, value); + } + + /** + * Converts key into a {@link CIKey} before calling {@link CIHashMap#getOrDefault(Object, Object)} + */ + public V getOrDefault(final String key, final V defaultValue) { + return super.getOrDefault(CIKey.of(key), defaultValue); + } + + /** + * Converts key into a {@link CIKey} before calling {@link CIHashMap#remove(String, Object)} + */ + public boolean remove(final String key, final Object value) { + return super.remove(CIKey.of(key), value); + } + + public boolean remove(final CIKey key, final Object value) { + return super.remove(key, value); + } + + /** + * Converts key into a {@link CIKey} before calling {@link CIHashMap#remove(Object)} + */ + public V remove(final String key) { + return super.remove(CIKey.of(key)); + } + + public V remove(final CIKey key) { + return super.remove(key); + } + + public boolean containsKey(final String key) { + return super.containsKey(CIKey.of(key)); + } + + public boolean containsKey(final CIKey key) { + return super.containsKey(key); + } + + @Override + public void putAll(final Map m) { + super.putAll(m); + } + + /** + * Equivalent to {@link Map#putAll(Map)} but for a map keyed by strings. + */ + public void putAllWithStringKeys(final Map map) { + GwtNullSafe.map(map) + .forEach(this::put); + } + + public Set keySetAsStrings() { + return keySet().stream() + .map(CIKey::get) + .collect(Collectors.toSet()); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static CIHashMap of(String k1, V v1) { + return new CIHashMap<>(CIKey.mapOf(k1, v1)); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static CIHashMap of(String k1, V v1, String k2, V v2) { + return new CIHashMap<>(CIKey.mapOf( + k1, v1, + k2, v2)); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static CIHashMap of(String k1, V v1, + String k2, V v2, + String k3, V v3) { + return new CIHashMap<>(CIKey.mapOf( + k1, v1, + k2, v2, + k3, v3)); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static CIHashMap of(String k1, V v1, + String k2, V v2, + String k3, V v3, + String k4, V v4) { + return new CIHashMap<>(CIKey.mapOf( + k1, v1, + k2, v2, + k3, v3, + k4, v4)); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static CIHashMap of(String k1, V v1, + String k2, V v2, + String k3, V v3, + String k4, V v4, + String k5, V v5) { + return new CIHashMap<>(CIKey.mapOf( + k1, v1, + k2, v2, + k3, v3, + k4, v4, + k5, v5)); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static CIHashMap of(String k1, V v1, + String k2, V v2, + String k3, V v3, + String k4, V v4, + String k5, V v5, + String k6, V v6) { + return new CIHashMap<>(CIKey.mapOf( + k1, v1, + k2, v2, + k3, v3, + k4, v4, + k5, v5, + k6, v6)); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static CIHashMap of(String k1, V v1, + String k2, V v2, + String k3, V v3, + String k4, V v4, + String k5, V v5, + String k6, V v6, + String k7, V v7) { + return new CIHashMap<>(CIKey.mapOf( + k1, v1, + k2, v2, + k3, v3, + k4, v4, + k5, v5, + k6, v6, + k7, v7)); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static CIHashMap of(String k1, V v1, + String k2, V v2, + String k3, V v3, + String k4, V v4, + String k5, V v5, + String k6, V v6, + String k7, V v7, + String k8, V v8) { + return new CIHashMap<>(CIKey.mapOf( + k1, v1, + k2, v2, + k3, v3, + k4, v4, + k5, v5, + k6, v6, + k7, v7, + k8, v8)); + } + + /** + * Convert an array of {@link String} keyed entries into a {@link CIKey} keyed map. + */ + @SafeVarargs + public static CIHashMap ofEntries(final Entry... entries) { + return GwtNullSafe.stream(entries) + .collect(Collectors.toMap( + entry -> + CIKey.of(entry.getKey()), + Entry::getValue, + (object, object2) -> object, + CIHashMap::new)); + } + + /** + * Convert a {@link String} keyed map into a {@link CIKey} keyed map. + * Accepts nulls and never returns a null. + */ + public static CIHashMap of(final Map map) { + return GwtNullSafe.map(map) + .entrySet() + .stream() + .collect(Collectors.toMap( + entry -> + CIKey.of(entry.getKey()), + Entry::getValue, + (object, object2) -> object, + CIHashMap::new)); + } +} diff --git a/stroom-util-shared/src/main/java/stroom/util/shared/string/CIHashSet.java b/stroom-util-shared/src/main/java/stroom/util/shared/string/CIHashSet.java new file mode 100644 index 00000000000..1abcb94b56f --- /dev/null +++ b/stroom-util-shared/src/main/java/stroom/util/shared/string/CIHashSet.java @@ -0,0 +1,53 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.shared.string; + +import stroom.util.shared.GwtNullSafe; + +import java.util.Collection; +import java.util.HashSet; +import java.util.stream.Collectors; + +/** + * A {@link HashSet} containing case-insensitive {@link CIKey} string values. + */ +public class CIHashSet extends HashSet { + + public CIHashSet(final Collection collection) { + super(GwtNullSafe.stream(collection) + .map(CIKey::of) + .collect(Collectors.toList())); + } + + public boolean add(final String value) { + return super.add(CIKey.of(value)); + } + + /** + * Remove an item matching value (case-insensitive). + */ + public boolean removeString(final String value) { + return super.remove(CIKey.of(value)); + } + + /** + * True if there is an item matching value (case-insensitive). + */ + public boolean containsString(final String value) { + return super.contains(CIKey.of(value)); + } +} diff --git a/stroom-util-shared/src/main/java/stroom/util/shared/string/CIKey.java b/stroom-util-shared/src/main/java/stroom/util/shared/string/CIKey.java new file mode 100644 index 00000000000..6721a8cc654 --- /dev/null +++ b/stroom-util-shared/src/main/java/stroom/util/shared/string/CIKey.java @@ -0,0 +1,598 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.shared.string; + +import stroom.util.shared.GwtNullSafe; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonValue; + +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A wrapper for a {@link String} whose {@link CIKey#equals(Object)} and + * {@link CIKey#hashCode()} methods are performed on the lower-case + * form of {@code key}. + *

+ * Useful as a case-insensitive cache key that retains the case of the + * original string at the cost of wrapping it in another object. Also + * useful for case insensitive comparisons of common strings. + *

+ *

+ * See {@link CIKeys} for common {@link CIKey} instances. + *

+ */ +@JsonInclude(Include.NON_NULL) +@JsonPropertyOrder(alphabetic = true) +public class CIKey implements Comparable { + + public static final CIKey NULL_STRING = new CIKey(null); + public static final CIKey EMPTY_STRING = new CIKey(""); + + // Compare on the lower case form of the key. CIKey may be null and lowerKey may be null + public static final Comparator COMPARATOR = Comparator.nullsFirst(Comparator.comparing( + CIKey::getAsLowerCase, + Comparator.nullsFirst(String::compareTo))); + + @JsonValue // No need to serialise the CIKey wrapper, just the key + private final String key; + + @JsonIgnore + private final transient String lowerKey; + + @JsonCreator + private CIKey(@JsonProperty("key") final String key) { + this.key = key; + this.lowerKey = toLowerCase(key); + } + + /** + * key and lowerKey must be equal ignoring case. + * + * @param key The key + * @param lowerKey The key converted to lower-case + */ + CIKey(final String key, final String lowerKey) { + this.key = key; + this.lowerKey = lowerKey; + } + + /** + * Create a {@link CIKey} for an unknown, upper or mixed case key, e.g. "FOO", or "Foo". + * If key is known to be all lower case then use {@link CIKey#ofLowerCase(String)}. + * If key is a common key this method will return an existing {@link CIKey} instance + * else it will create a new instance. + *

+ * The returned {@link CIKey} will wrap key with no change of case and no trimming. + *

+ */ + public static CIKey of(final String key) { + if (key == null) { + return NULL_STRING; + } else if (key.isEmpty()) { + return EMPTY_STRING; + } else { + // See if we have a common key that matches exactly with the one requested. + // Case-sensitive here because CIKey should wrap the exact case passed in. + return GwtNullSafe.requireNonNullElseGet( + CIKeys.KEY_TO_COMMON_CIKEY_MAP.get(key), + () -> new CIKey(key)); + } + } + + /** + * Equivalent to calling {@link CIKey#of(String)} with a trimmed key. + */ + public static CIKey trimmed(final String key) { + if (key == null) { + return NULL_STRING; + } else { + final String trimmed = key.trim(); + return CIKey.of(trimmed); + } + } + + /** + * Create a {@link CIKey} for an upper or mixed case key, e.g. "FOO", or "Foo", + * when you already know the lower-case form of the key. + * If key is all lower case then user {@link CIKey#ofLowerCase(String)}. + * If key is a common key this method will return an existing {@link CIKey} instance + * else it will create a new instance. + */ + public static CIKey of(final String key, final String lowerKey) { + if (key == null) { + return NULL_STRING; + } else if (key.isEmpty()) { + return EMPTY_STRING; + } else { + // See if we have a common key that matches exactly with the one requested. + // Case-sensitive here because CIKey should wrap the exact case passed in. + return GwtNullSafe.requireNonNullElseGet( + CIKeys.KEY_TO_COMMON_CIKEY_MAP.get(key), + () -> new CIKey(key, lowerKey)); + } + } + + /** + * Create a {@link CIKey} for key, providing a map of known {@link CIKey}s keyed + * on their key value. Allows callers to hold their own set of known {@link CIKey}s. + */ + public static CIKey of(final String key, final Map knownKeys) { + if (key == null) { + return NULL_STRING; + } else if (key.isEmpty()) { + return EMPTY_STRING; + } else { + CIKey ciKey = null; + if (knownKeys != null) { + ciKey = knownKeys.get(key); + } + if (ciKey == null) { + ciKey = CIKeys.KEY_TO_COMMON_CIKEY_MAP.get(key); + if (ciKey == null) { + ciKey = new CIKey(key); + } + } + return ciKey; + } + } + + /** + * Create a {@link CIKey} for an all lower case key, e.g. "foo". + * This is a minor optimisation to avoid a call to toLowerCase as the + * key is already in lower-case. + */ + public static CIKey ofLowerCase(final String lowerKey) { + if (lowerKey == null) { + return NULL_STRING; + } else if (lowerKey.isEmpty()) { + return EMPTY_STRING; + } else { + // See if we have a common key that matches exactly with the one requested. + // Case-sensitive here because CIKey should wrap the exact case passed in. + return GwtNullSafe.requireNonNullElseGet( + CIKeys.KEY_TO_COMMON_CIKEY_MAP.get(lowerKey), + () -> new CIKey(lowerKey, lowerKey)); + } + } + + /** + * Create a {@link CIKey} for a key that is known NOT to be in {@link CIKey}s map + * of common keys and is a key that will not be added to the map of common keys in future. + * This is a minor optimisation that saves a pointless map lookup. + */ + public static CIKey ofDynamicKey(final String dynamicKey) { + if (dynamicKey == null) { + return NULL_STRING; + } else if (dynamicKey.isEmpty()) { + return EMPTY_STRING; + } else { + return new CIKey(dynamicKey); + } + } + + /** + * Create a {@link CIKey} that will be held as a static variable. + * Only use this for commonly used static {@link CIKey} instances + * as if the key is not already held in the map of common {@link CIKey}s + * then it will be added. + */ + public static CIKey ofStaticKey(final String key) { + if (key == null) { + return NULL_STRING; + } else if (key.isEmpty()) { + return EMPTY_STRING; + } else { + // See if we have a common key that matches exactly with the one requested. + // Case-sensitive here because CIKey should wrap the exact case passed in. + return GwtNullSafe.requireNonNullElseGet( + CIKeys.KEY_TO_COMMON_CIKEY_MAP.get(key), + () -> CIKeys.commonKey(key)); + } + } + + /** + * If ciKey matches a common {@link CIKey} (ignoring case) then return the common + * {@link CIKey} else return ciKey. Use this if you don't care about the case of the + * wrapped string, e.g. if key is 'FOO', you could get back a {@link CIKey} that wraps + * 'foo', 'FOO', 'Foo', etc. + */ + public static CIKey ofIgnoringCase(final String key) { + if (key == null) { + return NULL_STRING; + } else if (key.isEmpty()) { + return EMPTY_STRING; + } else { + final String lowerKey = toLowerCase(key); + return GwtNullSafe.requireNonNullElseGet( + CIKeys.LOWER_KEY_TO_COMMON_CIKEY_MAP.get(lowerKey), + () -> CIKey.ofLowerCase(lowerKey)); + } + } + + /** + * @return The wrapped string in its original case. + */ + @JsonIgnore + public String get() { + return key; + } + + /** + * Here for JSON (de-)ser. + */ + private String getKey() { + return key; + } + + /** + * @return The lowercase form of the wrapped string. + */ + @JsonIgnore + public String getAsLowerCase() { + return lowerKey; + } + + /** + * @return True if str is equal to the string wrapped in this {@link CIKey}, ignoring case. + */ + public boolean equalsIgnoreCase(final String str) { + return CIKey.equalsIgnoreCase(str, this); + } + + /** + * Standard equals method for comparing two {@link CIKey} instances, comparing the + * lowerKey of each. + */ + @Override + public boolean equals(final Object object) { + if (this == object) { + return true; + } + if (object == null || getClass() != object.getClass()) { + return false; + } + final CIKey that = (CIKey) object; + return Objects.equals(lowerKey, that.lowerKey); + } + + @Override + public int hashCode() { + // String lazily caches its hashcode so no need for us to do it too + return lowerKey != null + ? lowerKey.hashCode() + : 0; + } + + @Override + public String toString() { + return key; + } + + @Override + public int compareTo(final CIKey o) { +// Objects.requireNonNull(o); + return COMPARATOR.compare(this, o); + } + + /** + * Returns true if the string this {@link CIKey} wraps contains subString + * ignoring case. + * If subString is all lower case, use {@link CIKey#containsLowerCase(String)} instead. + */ + public boolean containsIgnoreCase(final String subString) { + Objects.requireNonNull(subString); + if (lowerKey == null) { + return false; + } + return lowerKey.contains(toLowerCase(subString)); + } + + /** + * Returns true if the string this {@link CIKey} wraps contains (ignoring case) lowerSubString. + * {@code lowerSubString} MUST be all lower case. + *

+ * This method is a slight optimisation to avoid having to lower-case the input if it + * is know to already be lower-case. + *

+ * If lowerSubString is mixed or upper case, use {@link CIKey#containsIgnoreCase(String)} instead. + */ + public boolean containsLowerCase(final String lowerSubString) { + Objects.requireNonNull(lowerSubString); + if (lowerKey == null) { + return false; + } + return lowerKey.contains(toLowerCase(lowerSubString)); + } + + /** + * @param keys + * @return True if this key matches one of keys (ignoring case) + */ + public boolean in(final Collection keys) { + if (GwtNullSafe.hasItems(keys)) { + return keys.stream() + .anyMatch(aKey -> + CIKey.equalsIgnoreCase(this, aKey)); + } else { + return false; + } + } + + /** + * @return True if ciKey is null or wraps a null string + */ + public static boolean isNull(final CIKey ciKey) { + return ciKey == null || ciKey.key == null; + } + + public boolean isEmpty(final CIKey ciKey) { + return ciKey.key == null || ciKey.key.isEmpty(); + } + + public boolean isEmpty() { + return key == null || key.isEmpty(); + } + + /** + * Create a case-insensitive keyed {@link Entry} from a {@link String} key and value of type T. + */ + public static Entry entry(final String key, final V value) { + return Map.entry(CIKey.of(key), value); + } + + /** + * Create a case-insensitive keyed {@link Entry} from a simple {@link String} keyed {@link Entry}. + */ + public static Entry entry(final Entry entry) { + if (entry == null) { + return null; + } else { + return Map.entry(CIKey.of(entry.getKey()), entry.getValue()); + } + } + + public static List listOf(final String... keys) { + return GwtNullSafe.stream(keys) + .map(CIKey::of) + .collect(Collectors.toList()); + } + + public static Set setOf(final String... keys) { + return GwtNullSafe.stream(keys) + .map(CIKey::of) + .collect(Collectors.toSet()); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static Map mapOf(String k1, V v1) { + return Map.of(CIKey.of(k1), v1); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static Map mapOf(String k1, V v1, String k2, V v2) { + return Map.of( + CIKey.of(k1), v1, + CIKey.of(k2), v2); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static Map mapOf(String k1, V v1, + String k2, V v2, + String k3, V v3) { + return Map.of( + CIKey.of(k1), v1, + CIKey.of(k2), v2, + CIKey.of(k3), v3); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static Map mapOf(String k1, V v1, + String k2, V v2, + String k3, V v3, + String k4, V v4) { + return Map.of( + CIKey.of(k1), v1, + CIKey.of(k2), v2, + CIKey.of(k3), v3, + CIKey.of(k4), v4); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static Map mapOf(String k1, V v1, + String k2, V v2, + String k3, V v3, + String k4, V v4, + String k5, V v5) { + return Map.of( + CIKey.of(k1), v1, + CIKey.of(k2), v2, + CIKey.of(k3), v3, + CIKey.of(k4), v4, + CIKey.of(k5), v5); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static Map mapOf(String k1, V v1, + String k2, V v2, + String k3, V v3, + String k4, V v4, + String k5, V v5, + String k6, V v6) { + return Map.of( + CIKey.of(k1), v1, + CIKey.of(k2), v2, + CIKey.of(k3), v3, + CIKey.of(k4), v4, + CIKey.of(k5), v5, + CIKey.of(k6), v6); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static Map mapOf(String k1, V v1, + String k2, V v2, + String k3, V v3, + String k4, V v4, + String k5, V v5, + String k6, V v6, + String k7, V v7) { + return Map.of( + CIKey.of(k1), v1, + CIKey.of(k2), v2, + CIKey.of(k3), v3, + CIKey.of(k4), v4, + CIKey.of(k5), v5, + CIKey.of(k6), v6, + CIKey.of(k7), v7); + } + + /** + * Create a {@link CIKey} keyed map + */ + public static Map mapOf(String k1, V v1, + String k2, V v2, + String k3, V v3, + String k4, V v4, + String k5, V v5, + String k6, V v6, + String k7, V v7, + String k8, V v8) { + return Map.of( + CIKey.of(k1), v1, + CIKey.of(k2), v2, + CIKey.of(k3), v3, + CIKey.of(k4), v4, + CIKey.of(k5), v5, + CIKey.of(k6), v6, + CIKey.of(k7), v7, + CIKey.of(k8), v8); + } + + /** + * Convert an array of {@link String} keyed entries into a {@link CIKey} keyed map. + */ + @SafeVarargs + public static Map mapOfEntries(final Entry... entries) { + return GwtNullSafe.stream(entries) + .collect(Collectors.toMap( + entry -> + CIKey.of(entry.getKey()), + Entry::getValue)); + } + + /** + * Convert a {@link String} keyed map into a {@link CIKey} keyed map. + * Accepts nulls and never returns a null. + */ + public static Map mapOf(final Map map) { + return GwtNullSafe.map(map) + .entrySet() + .stream() + .collect(Collectors.toMap( + entry -> + CIKey.of(entry.getKey()), + Entry::getValue)); + } + + public static Map convertToStringMap(final Map map) { + return GwtNullSafe.map(map) + .entrySet() + .stream() + .collect(Collectors.toMap( + entry -> entry.getKey().get(), + Entry::getValue)); + } + + public static Map convertToLowerCaseStringMap(final Map map) { + return GwtNullSafe.map(map) + .entrySet() + .stream() + .collect(Collectors.toMap( + entry -> entry.getKey().getAsLowerCase(), + Entry::getValue)); + } + + public static V put(final Map map, + final String key, + final V value) { + return map.put(CIKey.of(key), value); + } + + /** + * True if str is equal to the string wrapped by ciKey, ignoring case. + */ + public static boolean equalsIgnoreCase(final String str1, final String str2) { + if (str1 == null && str2 == null) { + return true; + } else { + return str1 != null && str1.equalsIgnoreCase(str2); + } + } + + /** + * True if str is equal to the string wrapped by ciKey, ignoring case. + */ + public static boolean equalsIgnoreCase(final CIKey ciKey, final String str) { + return equalsIgnoreCase(str, ciKey); + } + + /** + * True if str is equal to the string wrapped by ciKey, ignoring case. + */ + public static boolean equalsIgnoreCase(final String str, final CIKey ciKey) { + final String lowerKey = ciKey.lowerKey; + if (lowerKey == null && str == null) { + return true; + } else { + return lowerKey != null && lowerKey.equalsIgnoreCase(str); + } + } + + /** + * Method so we have a consistent way of doing it, in the unlikely event it changes. + */ + static String toLowerCase(final String str) { + return GwtNullSafe.get(str, s -> s.toLowerCase(Locale.ENGLISH)); + } +} diff --git a/stroom-util-shared/src/main/java/stroom/util/shared/string/CIKeys.java b/stroom-util-shared/src/main/java/stroom/util/shared/string/CIKeys.java new file mode 100644 index 00000000000..4c49968c0da --- /dev/null +++ b/stroom-util-shared/src/main/java/stroom/util/shared/string/CIKeys.java @@ -0,0 +1,208 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.shared.string; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * A set of static common {@link CIKey} instances + */ +public class CIKeys { + + // Hold some common keys, so we can just re-use instances rather than creating new each time + // Some of these map values will get used statically in code, but a lot will come from dash/query + // fields/columns/tokens which are not known at compile time so the map lets us save on + // object creation for common ones. + // The cost of a hashmap get is less than the combined cost of CIKey object creation and + // the toLowerCase call. + // Have to be concurrent maps as various classes in potentially multiple threads will call + // commonKey(). Not much perf difference as compared to a HashMap in tests + static final Map KEY_TO_COMMON_CIKEY_MAP = new ConcurrentHashMap<>(); + static final Map LOWER_KEY_TO_COMMON_CIKEY_MAP = new ConcurrentHashMap<>(); + + // Upper camel case keys + public static final CIKey AUTHORIZATION = commonKey("Authorization"); + public static final CIKey COMPRESSION = commonKey("Compression"); + public static final CIKey DURATION = commonKey("Duration"); + public static final CIKey EFFECTIVE_TIME = commonKey("EffectiveTime"); + public static final CIKey END = commonKey("End"); + public static final CIKey EVENT_ID = commonKey("EventId"); + public static final CIKey EVENT_TIME = commonKey("EventTime"); + public static final CIKey FEED = commonKey("Feed"); + public static final CIKey FORWARD_ERROR = commonKey("ForwardError"); + public static final CIKey FILES = commonKey("Files"); + public static final CIKey GUID = commonKey("GUID"); + public static final CIKey ID = commonKey("Id"); + public static final CIKey INDEX = commonKey("Index"); + public static final CIKey INSERT_TIME = commonKey("InsertTime"); + public static final CIKey KEY = commonKey("Key"); + public static final CIKey KEY_END = commonKey("KeyEnd"); + public static final CIKey KEY_START = commonKey("KeyStart"); + public static final CIKey NAME = commonKey("Name"); + public static final CIKey NODE = commonKey("Node"); + public static final CIKey PARTITION = commonKey("Partition"); + public static final CIKey PIPELINE = commonKey("Pipeline"); + public static final CIKey PROXY_FORWARD_ID = commonKey("ProxyForwardId"); + public static final CIKey RECEIVED_PATH = commonKey("ReceivedPath"); + public static final CIKey RECEIVED_TIME = commonKey("ReceivedTime"); + public static final CIKey RECEIVED_TIME_HISTORY = commonKey("ReceivedTimeHistory"); + public static final CIKey REMOTE_ADDRESS = commonKey("RemoteAddress"); + public static final CIKey REMOTE_CERT_EXPIRY = commonKey("RemoteCertExpiry"); + public static final CIKey REMOTE_FILE = commonKey("RemoteFile"); + public static final CIKey REMOTE_HOST = commonKey("RemoteHost"); + public static final CIKey REMOTE_DN = commonKey("RemoteDn"); + public static final CIKey START = commonKey("Start"); + public static final CIKey STATUS = commonKey("Status"); + public static final CIKey STREAM_ID = commonKey("StreamId"); + public static final CIKey STREAM_SIZE = commonKey("StreamSize"); + public static final CIKey SUBJECT = commonKey("Subject"); + public static final CIKey TERMINAL = commonKey("Terminal"); + public static final CIKey TIME = commonKey("Time"); + public static final CIKey TITLE = commonKey("Title"); + public static final CIKey TYPE = commonKey("Type"); + public static final CIKey UPLOAD_USERNAME = commonKey("UploadUsername"); + public static final CIKey UPLOAD_USER_ID = commonKey("UploadUserId"); + public static final CIKey USER = commonKey("User"); + public static final CIKey UUID = commonKey("UUID"); + public static final CIKey VALUE = commonKey("Value"); + public static final CIKey VALUE_TYPE = commonKey("ValueType"); + + // Lower case keys + public static final CIKey ACCEPT = commonKey("accept"); + public static final CIKey CONNECTION = commonKey("connection"); + public static final CIKey EXPECT = commonKey("expect"); + + // kebab case keys + public static final CIKey CONTENT___ENCODING = commonKey("content-encoding"); + public static final CIKey CONTENT___LENGTH = commonKey("content-length"); + public static final CIKey TRANSFER___ENCODING = commonKey("transfer-encoding"); + public static final CIKey USER___AGENT = commonKey("user-agent"); + public static final CIKey X___FORWARDED___FOR = commonKey("X-Forwarded-For"); + + // Upper sentence case keys + public static final CIKey ANALYTIC__RULE = commonKey("Analytic Rule"); + public static final CIKey CREATE__TIME = commonKey("Create Time"); + public static final CIKey CREATE__TIME__MS = commonKey("Create Time Ms"); + public static final CIKey DOC__COUNT = commonKey("Doc Count"); + public static final CIKey EFFECTIVE__TIME = commonKey("Effective Time"); + public static final CIKey END__TIME = commonKey("End Time"); + public static final CIKey END__TIME__MS = commonKey("End Time Ms"); + public static final CIKey ERROR__COUNT = commonKey("Error Count"); + public static final CIKey FATAL__ERROR__COUNT = commonKey("Fatal Error Count"); + public static final CIKey FILE__SIZE = commonKey("File Size"); + public static final CIKey INDEX__NAME = commonKey("Index Name"); + public static final CIKey INFO__COUNT = commonKey("Info Count"); + public static final CIKey LAST__COMMIT = commonKey("Last Commit"); + public static final CIKey META__ID = commonKey("Meta Id"); + public static final CIKey PARENT__CREATE__TIME = commonKey("Parent Create Time"); + public static final CIKey PARENT__FEED = commonKey("Parent Feed"); + public static final CIKey PARENT__ID = commonKey("Parent Id"); + public static final CIKey PARENT__STATUS = commonKey("Parent Status"); + public static final CIKey PIPELINE__NAME = commonKey("Pipeline Name"); + public static final CIKey PROCESSOR__DELETED = commonKey("Processor Deleted"); + public static final CIKey PROCESSOR__ENABLED = commonKey("Processor Enabled"); + public static final CIKey PROCESSOR__FILTER__DELETED = commonKey("Processor Filter Deleted"); + public static final CIKey PROCESSOR__FILTER__ENABLED = commonKey("Processor Filter Enabled"); + public static final CIKey PROCESSOR__FILTER__ID = commonKey("Processor Filter Id"); + public static final CIKey PROCESSOR__FILTER__LAST__POLL__MS = commonKey("Processor Filter Last Poll Ms"); + public static final CIKey PROCESSOR__FILTER__PRIORITY = commonKey("Processor Filter Priority"); + public static final CIKey PROCESSOR__FILTER__UUID = commonKey("Processor Filter UUID"); + public static final CIKey PROCESSOR__ID = commonKey("Processor Id"); + public static final CIKey PROCESSOR__PIPELINE = commonKey("Processor Pipeline"); + public static final CIKey PROCESSOR__TASK__ID = commonKey("Processor Task Id"); + public static final CIKey PROCESSOR__TYPE = commonKey("Processor Type"); + public static final CIKey PROCESSOR__UUID = commonKey("Processor UUID"); + public static final CIKey RAW__SIZE = commonKey("Raw Size"); + public static final CIKey READ__COUNT = commonKey("Read Count"); + public static final CIKey START__TIME = commonKey("Start Time"); + public static final CIKey START__TIME__MS = commonKey("Start Time Ms"); + public static final CIKey STATUS__TIME = commonKey("Status Time"); + public static final CIKey STATUS__TIME__MS = commonKey("Status Time Ms"); + public static final CIKey TASK__ID = commonKey("Task Id"); + public static final CIKey VOLUME__GROUP = commonKey("Volume Group"); + public static final CIKey VOLUME__PATH = commonKey("Volume Path"); + public static final CIKey WARNING__COUNT = commonKey("Warning Count"); + public static final CIKey WRITE__COUNT = commonKey("Write Count"); + + // Reference Data fields + public static final CIKey FEED__NAME = commonKey("Feed Name"); + public static final CIKey LAST__ACCESSED__TIME = commonKey("Last Accessed Time"); + public static final CIKey MAP__NAME = commonKey("Map Name"); + public static final CIKey PART__NUMBER = commonKey("Part Number"); + public static final CIKey PIPELINE__VERSION = commonKey("Pipeline Version"); + public static final CIKey PROCESSING__STATE = commonKey("Processing State"); + public static final CIKey REFERENCE__LOADER__PIPELINE = commonKey("Reference Loader Pipeline"); + public static final CIKey STREAM__ID = commonKey("Stream ID"); + public static final CIKey VALUE__REFERENCE__COUNT = commonKey("Value Reference Count"); + + // Annotations keys + public static final CIKey ANNO_ASSIGNED_TO = commonKey("annotation:AssignedTo"); + public static final CIKey ANNO_COMMENT = commonKey("annotation:Comment"); + public static final CIKey ANNO_CREATED_BY = commonKey("annotation:CreatedBy"); + public static final CIKey ANNO_CREATED_ON = commonKey("annotation:CreatedOn"); + public static final CIKey ANNO_HISTORY = commonKey("annotation:History"); + public static final CIKey ANNO_ID = commonKey("annotation:Id"); + public static final CIKey ANNO_STATUS = commonKey("annotation:Status"); + public static final CIKey ANNO_SUBJECT = commonKey("annotation:Subject"); + public static final CIKey ANNO_TITLE = commonKey("annotation:Title"); + public static final CIKey ANNO_UPDATED_BY = commonKey("annotation:UpdatedBy"); + public static final CIKey ANNO_UPDATED_ON = commonKey("annotation:UpdatedOn"); + + public static final CIKey UNDERSCORE_EVENT_ID = commonKey("__event_id__"); + public static final CIKey UNDERSCORE_STREAM_ID = commonKey("__stream_id__"); + public static final CIKey UNDERSCORE_TIME = commonKey("__time__"); + + private CIKeys() { + } + + /** + * Add key to the map of common keys for re-use. + *

+ * Only intended for use on static and commonly used {@link CIKey} instances due to the cost of + * string interning and storage. + *

+ */ + static CIKey commonKey(final String key) { + final CIKey ciKey; + if (key == null) { + ciKey = CIKey.NULL_STRING; + } else if (key.isEmpty()) { + ciKey = CIKey.EMPTY_STRING; + } else { + // Ensure we are using string pool instances for both + final String k = key.intern(); + ciKey = new CIKey(k, CIKey.toLowerCase(k).intern()); + } + // Add it to our static maps, so we can get a common CIKey either from its + // exact case or its lower case form. + CIKeys.KEY_TO_COMMON_CIKEY_MAP.put(key, ciKey); + CIKeys.LOWER_KEY_TO_COMMON_CIKEY_MAP.put(ciKey.getAsLowerCase(), ciKey); + return ciKey; + } + + /** + * @return The set of all common {@link CIKey}s + */ + static Set commonKeys() { + //noinspection Java9CollectionFactory // GWT :-( + return Collections.unmodifiableSet(new HashSet<>(KEY_TO_COMMON_CIKEY_MAP.values())); + } +} diff --git a/stroom-util-shared/src/test/java/stroom/util/shared/query/TestFieldNames.java b/stroom-util-shared/src/test/java/stroom/util/shared/query/TestFieldNames.java new file mode 100644 index 00000000000..58bafc7c46b --- /dev/null +++ b/stroom-util-shared/src/test/java/stroom/util/shared/query/TestFieldNames.java @@ -0,0 +1,84 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.shared.query; + +import stroom.test.common.TestUtil; +import stroom.util.shared.string.CIKey; + +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static stroom.util.shared.query.FieldNames.DEFAULT_EVENT_ID_FIELD_NAME; +import static stroom.util.shared.query.FieldNames.DEFAULT_STREAM_ID_FIELD_NAME; +import static stroom.util.shared.query.FieldNames.DEFAULT_TIME_FIELD_NAME; +import static stroom.util.shared.query.FieldNames.FALLBACK_EVENT_ID_FIELD_NAME; +import static stroom.util.shared.query.FieldNames.FALLBACK_STREAM_ID_FIELD_NAME; +import static stroom.util.shared.query.FieldNames.FALLBACK_TIME_FIELD_NAME; + +class TestFieldNames { + + @Test + void isStreamIdFieldName() { + assertThat(FieldNames.isStreamIdFieldName("foo")) + .isFalse(); + assertThat(FieldNames.isStreamIdFieldName(DEFAULT_STREAM_ID_FIELD_NAME)) + .isTrue(); + assertThat(FieldNames.isStreamIdFieldName(FALLBACK_STREAM_ID_FIELD_NAME)) + .isTrue(); + } + + @Test + void isEventIdFieldName() { + assertThat(FieldNames.isEventIdFieldName("foo")) + .isFalse(); + assertThat(FieldNames.isEventIdFieldName(DEFAULT_EVENT_ID_FIELD_NAME)) + .isTrue(); + assertThat(FieldNames.isEventIdFieldName(FALLBACK_EVENT_ID_FIELD_NAME)) + .isTrue(); + } + + @TestFactory + Stream testCreateCIKey() { + return TestUtil.buildDynamicTestStream() + .withInputType(String.class) + .withOutputType(CIKey.class) + .withSingleArgTestFunction(CIKey::of) + .withAssertions(outcome -> { + final String input = outcome.getInput(); + final CIKey actualOutput = outcome.getActualOutput(); + assertThat(actualOutput.get()) + .isEqualTo(input); + assertThat(actualOutput) + .isSameAs(outcome.getExpectedOutput()); + }) + // Not using expected output + .addCase(DEFAULT_TIME_FIELD_NAME, FieldNames.DEFAULT_TIME_FIELD_KEY) + .addCase(FALLBACK_TIME_FIELD_NAME, FieldNames.FALLBACK_TIME_FIELD_KEY) + .addCase(DEFAULT_STREAM_ID_FIELD_NAME, FieldNames.DEFAULT_STREAM_ID_FIELD_KEY) + .addCase(FALLBACK_STREAM_ID_FIELD_NAME, FieldNames.FALLBACK_STREAM_ID_FIELD_KEY) + .addCase(DEFAULT_EVENT_ID_FIELD_NAME, FieldNames.DEFAULT_EVENT_ID_FIELD_KEY) + .addCase(FALLBACK_EVENT_ID_FIELD_NAME, FieldNames.FALLBACK_EVENT_ID_FIELD_KEY) + .addCase(FALLBACK_EVENT_ID_FIELD_NAME, FieldNames.FALLBACK_EVENT_ID_FIELD_KEY) + .addCase(null, CIKey.NULL_STRING) + .addCase("", CIKey.EMPTY_STRING) + .build(); + } +} diff --git a/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIHashMap.java b/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIHashMap.java new file mode 100644 index 00000000000..b3eb406f025 --- /dev/null +++ b/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIHashMap.java @@ -0,0 +1,107 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.shared.string; + +import org.junit.jupiter.api.Test; + +import static java.util.Map.Entry.comparingByKey; +import static org.assertj.core.api.Assertions.assertThat; + +class TestCIHashMap { + + @Test + void putAndGet() { + final CIHashMap map = new CIHashMap<>(); + // Same effective key + map.put("foo", "A"); + map.put(CIKey.of("FOO"), "B"); + + // Same effective key + map.put("bar", "C"); + map.put("BAR", "D"); + + assertThat(map.size()) + .isEqualTo(2); + + assertThat(map.get("foo")) + .isEqualTo("B"); + assertThat(map.get("foo")) + .isEqualTo(map.get(CIKey.of("Foo"))); + + assertThat(map.get("bar")) + .isEqualTo("D"); + assertThat(map.get("bar")) + .isEqualTo(map.get(CIKey.of("Bar"))); + + assertThat(map.containsKey("foo")) + .isTrue(); + assertThat(map.containsKey(CIKey.of("foo"))) + .isTrue(); + + assertThat(map.containsKey("bar")) + .isTrue(); + assertThat(map.containsKey(CIKey.of("bar"))) + .isTrue(); + } + + @Test + void sorting() { + final CIHashMap map = new CIHashMap<>(); + map.put("D", "1"); + map.put("e", "1"); + map.put("B", "1"); + map.put("c", "1"); + map.put("a", "1"); + + assertThat(map.entrySet() + .stream() + .sorted(comparingByKey()) + .toList()) + .extracting(entry -> entry.getKey().getAsLowerCase()) + .containsExactly( + "a", + "b", + "c", + "d", + "e"); + } + + @Test + void testContainsKey() { + final CIHashMap map = new CIHashMap<>(CIKey.mapOf( + "a", 100, + "b", 200)); + + assertThat(map.containsKey("a")) + .isTrue(); + assertThat(map.containsKey("A")) + .isTrue(); + assertThat(map.containsKey("B")) + .isTrue(); + assertThat(map.containsKey("z")) + .isFalse(); + + assertThat(map.containsKey(CIKey.of("a"))) + .isTrue(); + assertThat(map.containsKey(CIKey.of("A"))) + .isTrue(); + assertThat(map.containsKey(CIKey.of("B"))) + .isTrue(); + assertThat(map.containsKey(CIKey.of("z"))) + .isFalse(); + } +} diff --git a/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIKey.java b/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIKey.java new file mode 100644 index 00000000000..062269b3bcf --- /dev/null +++ b/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIKey.java @@ -0,0 +1,527 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.shared.string; + +import stroom.test.common.TestUtil; +import stroom.test.common.TestUtil.TimedCase; +import stroom.util.json.JsonUtil; +import stroom.util.logging.LambdaLogger; +import stroom.util.logging.LambdaLoggerFactory; +import stroom.util.shared.CompareUtil; +import stroom.util.shared.GwtNullSafe; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.inject.TypeLiteral; +import io.vavr.Tuple; +import io.vavr.Tuple2; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static stroom.util.shared.string.CIKey.equalsIgnoreCase; +import static stroom.util.shared.string.CIKey.listOf; + +public class TestCIKey { + + private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(TestCIKey.class); + + @TestFactory + Stream test() { + return TestUtil.buildDynamicTestStream() + .withInputTypes(String.class, String.class) + .withOutputType(boolean.class) + .withTestFunction(testCase -> { + final CIKey str1 = CIKey.of(testCase.getInput()._1()); + final CIKey str2 = CIKey.of(testCase.getInput()._2()); + // Make sure the wrappers hold the original value + assertThat(str1.get()) + .isEqualTo(testCase.getInput()._1()); + assertThat(str2.get()) + .isEqualTo(testCase.getInput()._2()); + + final boolean areEqual = Objects.equals(str1, str2); + final boolean haveEqualHashCode = Objects.equals(str1.hashCode(), str2.hashCode()); + + // If objects are equal, so should the hashes + assertThat(haveEqualHashCode) + .isEqualTo(areEqual); + + return areEqual; + }) + .withSimpleEqualityAssertion() + .addCase(Tuple.of(null, null), true) + .addCase(Tuple.of(null, "foo"), false) + .addCase(Tuple.of("foo", null), false) + .addCase(Tuple.of("foo", "bar"), false) + .addCase(Tuple.of("foo", "foox"), false) + .addCase(Tuple.of("foo", "foo"), true) + .addCase(Tuple.of("foo", "FOO"), true) + .addCase(Tuple.of("foo", "Foo"), true) + .addCase(Tuple.of("foo123", "Foo123"), true) + .addCase(Tuple.of("123", "123"), true) + .addCase(Tuple.of("", ""), true) + .build(); + } + + @Test + void testWithMap() { + + final Map map = new HashMap<>(); + + final Consumer putter = str -> + map.put(CIKey.of(str), str); + + putter.accept("foo"); // first key put to 'foo' + putter.accept("fOo"); + putter.accept("FOO"); // Last value put to 'foo' + putter.accept("bar"); + + assertThat(map) + .hasSize(2); + assertThat(map) + .containsKeys( + CIKey.of("foo"), + CIKey.of("bar")); + + assertThat(map.keySet().stream().map(CIKey::get).collect(Collectors.toSet())) + .contains("foo", "bar"); + + assertThat(map.values()) + .contains("FOO", "bar"); + + assertThat(map.get(CIKey.of("foo"))) + .isEqualTo("FOO"); + assertThat(map.get(CIKey.of("FOO"))) + .isEqualTo("FOO"); + } + + @TestFactory + Stream testEqualsIgnoreCase() { + return TestUtil.buildDynamicTestStream() + .withInputTypes(String.class, String.class) + .withOutputType(boolean.class) + .withTestFunction(testCase -> { + final String str = testCase.getInput()._1; + final CIKey ciKey = CIKey.of(testCase.getInput()._2); + boolean isEqual = equalsIgnoreCase(str, ciKey); + // Test the other overloaded equalsIgnoreCase methods too + assertThat(equalsIgnoreCase(ciKey, str)) + .isEqualTo(isEqual); + assertThat(equalsIgnoreCase(str, ciKey.get())) + .isEqualTo(isEqual); + return isEqual; + }) + .withSimpleEqualityAssertion() + .addCase(Tuple.of(null, null), true) + .addCase(Tuple.of("", ""), true) + .addCase(Tuple.of("foo", "foo"), true) + .addCase(Tuple.of("foo", "FOO"), true) + .addCase(Tuple.of("FOO", "foo"), true) + .addCase(Tuple.of("foo", null), false) + .addCase(Tuple.of(null, "foo"), false) + .addCase(Tuple.of("foo", "bar"), false) + .build(); + } + + @TestFactory + Stream testContainsIgnoreCase() { + return TestUtil.buildDynamicTestStream() + .withInputTypes(String.class, String.class) + .withOutputType(boolean.class) + .withTestFunction(testCase -> { + final String str = testCase.getInput()._1; + final String subStr = testCase.getInput()._2; + final CIKey ciKey = CIKey.of(str); + return ciKey.containsIgnoreCase(subStr); + }) + .withSimpleEqualityAssertion() + .addCase(Tuple.of("", ""), true) + .addCase(Tuple.of("foo", "f"), true) + .addCase(Tuple.of("foo", "oo"), true) + .addCase(Tuple.of("FOO", "f"), true) + .addCase(Tuple.of("FOO", "oo"), true) + .addCase(Tuple.of("foo", "F"), true) + .addCase(Tuple.of("foo", "OO"), true) + .addCase(Tuple.of("foo", "x"), false) + .addCase(Tuple.of("FOO", "x"), false) + .addCase(Tuple.of("foo", "X"), false) + .build(); + } + + @TestFactory + Stream testContainsLowerCase() { + return TestUtil.buildDynamicTestStream() + .withInputTypes(String.class, String.class) + .withOutputType(boolean.class) + .withTestFunction(testCase -> { + final String str = testCase.getInput()._1; + final String subStr = testCase.getInput()._2; + final CIKey ciKey = CIKey.of(str); + return ciKey.containsLowerCase(subStr); + }) + .withSimpleEqualityAssertion() + .addCase(Tuple.of("", ""), true) + .addCase(Tuple.of("foo", "f"), true) + .addCase(Tuple.of("foo", "oo"), true) + .addCase(Tuple.of("FOO", "f"), true) + .addCase(Tuple.of("FOO", "oo"), true) + .addCase(Tuple.of("foo", "x"), false) + .addCase(Tuple.of("FOO", "x"), false) + .build(); + } + + @TestFactory + Stream testIn() { + return TestUtil.buildDynamicTestStream() + .withWrappedInputType(new TypeLiteral>>() { + }) + .withOutputType(boolean.class) + .withTestFunction(testCase -> { + final String key = testCase.getInput()._1; + final List keys = testCase.getInput()._2; + return CIKey.of(key).in(keys); + }) + .withSimpleEqualityAssertion() + .addCase(Tuple.of("foo", List.of("foo", "bar")), true) + .addCase(Tuple.of("bar", List.of("foo", "bar")), true) + .addCase(Tuple.of("BAR", List.of("foo", "bar")), true) + .addCase(Tuple.of("FOO", List.of("foo", "bar")), true) + .addCase(Tuple.of("xxx", List.of("foo", "bar")), false) + .addCase(Tuple.of("", List.of("foo", "bar")), false) + .addCase(Tuple.of("", List.of("foo", "", "bar")), true) + .addCase(Tuple.of(null, Arrays.asList("foo", "bar", null)), true) + .build(); + } + + @Test + void testListOf() { + assertThat(listOf("a", "B", "c")) + .extracting(CIKey::getAsLowerCase) + .containsExactly("a", "b", "c"); + + assertThat(listOf((String[]) null)) + .extracting(CIKey::getAsLowerCase) + .isEmpty(); + + assertThat(listOf()) + .extracting(CIKey::getAsLowerCase) + .isEmpty(); + } + + @Test + void testSorting() { + + final Map map = new HashMap<>(); + Stream.of("A", "aa", "b", "C", "d", null, "", "1", "0") + .forEach(str -> map.put(CIKey.of(str), str)); + + assertThat(map.keySet() + .stream() + .sorted(Comparator.nullsFirst(CIKey.COMPARATOR)) + .toList()) + .extracting(CIKey::get) + .containsExactly(null, "", "0", "1", "A", "aa", "b", "C", "d"); + + assertThat(map.keySet() + .stream() + .sorted() + .toList()) + .extracting(CIKey::getAsLowerCase) + .containsExactly(null, "", "0", "1", "a", "aa", "b", "c", "d"); + } + + @Test + void testWithKnownKeys() { + final Map knownCIKeys = Stream.of( + "Foo", + "Bar") + .map(CIKey::of) + .collect(Collectors.toMap( + CIKey::get, + Function.identity())); + + final CIKey knownCIKey = knownCIKeys.get("Foo"); + + final CIKey ciKey = CIKey.of("Foo", knownCIKeys); + assertThat(ciKey) + .isSameAs(knownCIKey); + + // Different case so not known + final CIKey ciKey2 = CIKey.of("foo", knownCIKeys); + assertThat(ciKey2) + .isNotSameAs(knownCIKey); + } + + @Test + void testWithKnownKeys2() { + final Map knownCIKeys = Stream.of( + "Foo", + "Bar") + .map(CIKey::of) + .collect(Collectors.toMap( + CIKey::get, + Function.identity())); + + // Not in known keys, so uses one from built-in common keys + final CIKey ciKey = CIKey.of(CIKeys.UUID.get(), knownCIKeys); + assertThat(ciKey) + .isSameAs(CIKeys.UUID); + } + + @Test + void testWithCommonKey() { + // Not in known keys, so uses one from built-in common keys + final CIKey ciKey = CIKey.of(CIKeys.UUID.get()); + assertThat(ciKey) + .isSameAs(CIKeys.UUID); + } + + @Test + void testSerialisation() throws JsonProcessingException { + final CIKey ciKey1 = CIKey.of("foo"); + final CIKey ciKey2 = CIKey.of("bar"); + String json = JsonUtil.getMapper() + .writeValueAsString(ciKey1); + LOGGER.info("json\n{}", json); + + assertThat(json) + .isEqualTo(""" + "foo\""""); + + final CIKey ciKey = JsonUtil.getMapper().readValue(json, CIKey.class); + assertThat(ciKey) + .isEqualTo(ciKey1); + + final Map map = Map.of( + ciKey1, "A"); + + json = JsonUtil.getMapper() + .writeValueAsString(map); + LOGGER.info("json\n{}", json); + assertThat(json) + .isEqualTo(""" + { + "foo" : "A" + }"""); + + final Map map2 = JsonUtil.getMapper().readValue(json, new TypeReference<>() { + }); + assertThat(map2) + .isEqualTo(map); + } + + @Test + void trimmed() { + CIKey ciKey1 = CIKey.trimmed(" Foo "); + CIKey ciKey2 = CIKey.of("Foo"); + + assertThat(ciKey1) + .isEqualTo(ciKey2); + assertThat(ciKey1) + .isNotSameAs(ciKey2); + assertThat(ciKey1.get()) + .isEqualTo("Foo"); + assertThat(ciKey1.getAsLowerCase()) + .isEqualTo("foo"); + } + + @Test + void testOfLowerCase() { + final String key = "foo"; + CIKey ciKey = CIKey.ofLowerCase(key); + assertThat(ciKey.get()) + .isSameAs(key); + assertThat(ciKey.getAsLowerCase()) + .isSameAs(key); + } + + @Test + void testOfDynamicKey() { + final String key = "UUID"; + CIKey ciKey1 = CIKeys.UUID; + CIKey ciKey2 = CIKey.of(key); + CIKey ciKey3 = CIKey.ofDynamicKey(key); + CIKey ciKey4 = CIKey.ofDynamicKey(key); + + assertThat(ciKey1) + .isSameAs(ciKey2); + assertThat(ciKey1) + .isNotSameAs(ciKey3); + assertThat(ciKey1) + .isNotSameAs(ciKey4); + + assertThat(ciKey1) + .isEqualTo(ciKey3); + assertThat(ciKey1) + .isEqualTo(ciKey4); + } + + @TestFactory + Stream testComparator() { + return TestUtil.buildDynamicTestStream() + .withInputTypes(CIKey.class, CIKey.class) + .withOutputType(int.class) + .withTestFunction(testCase -> { + final int result = CompareUtil.normalise(CIKey.COMPARATOR.compare( + testCase.getInput()._1, + testCase.getInput()._2)); + + // Reverse it + final int result2 = CompareUtil.normalise(CIKey.COMPARATOR.compare( + testCase.getInput()._2, + testCase.getInput()._1)); + + if (result == 0) { + Assertions.assertThat(result2) + .isEqualTo(0); + } else { + Assertions.assertThat(result2) + .isEqualTo(-1 * result); + } + return result; + }) + .withSimpleEqualityAssertion() + .addCase(Tuple.of(null, CIKey.ofDynamicKey("a")), -1) + .addCase(Tuple.of(CIKey.NULL_STRING, CIKey.EMPTY_STRING), -1) + .addCase(Tuple.of(CIKey.NULL_STRING, CIKey.ofDynamicKey("a")), -1) + .addCase(Tuple.of(CIKey.NULL_STRING, CIKey.EMPTY_STRING), -1) + .addCase(Tuple.of(CIKey.ofDynamicKey("aaa"), CIKey.ofDynamicKey("bbb")), -1) + .addCase(Tuple.of(CIKey.ofDynamicKey("aaa"), CIKey.ofDynamicKey("BBB")), -1) + .addCase(Tuple.of(CIKey.ofDynamicKey("aaa"), CIKey.ofDynamicKey("AAA")), 0) + .addCase(Tuple.of(CIKey.ofDynamicKey("a"), CIKey.ofDynamicKey("aaa")), -1) + .build(); + } + + @Test + @Disabled + // manual run only + void testPerf() { + final List keys = new ArrayList<>(CIKeys.commonKeys() + .stream() + .map(CIKey::get) + .toList()); + keys.add(CIKey.EMPTY_STRING.get()); + keys.add(null); + + final List lowerKeys = keys.stream() + .map(key -> GwtNullSafe.get(key, String::toLowerCase)) + .toList(); + + final int cpuCount = Runtime.getRuntime().availableProcessors(); + final ExecutorService executorService = Executors.newFixedThreadPool(cpuCount); + + // Will always get the CIKey instance from a map. + final TimedCase commonKeyCase = TimedCase.of("Common Key", (round, iterations) -> { + doWorkOnThreads(cpuCount, iterations, executorService, () -> { + for (int j = 0; j < keys.size(); j++) { + final String key = keys.get(j); + final String lowerKey = lowerKeys.get(j); + final CIKey ciKey = CIKey.of(key); + if (!CIKey.equalsIgnoreCase(ciKey, lowerKey)) { + throw new RuntimeException("Mismatch"); + } + } + }); + }); + + // Will always create a new CIKey instance, except for "" and null. + final TimedCase dynamicKeyCase = TimedCase.of("Dynamic Key", (round, iterations) -> { + doWorkOnThreads(cpuCount, iterations, executorService, () -> { + for (int j = 0; j < keys.size(); j++) { + final String key = keys.get(j); + final String lowerKey = lowerKeys.get(j); + final CIKey ciKey = CIKey.ofDynamicKey(key); + if (!CIKey.equalsIgnoreCase(ciKey, lowerKey)) { + throw new RuntimeException("Mismatch"); + } + } + }); + }); + + // No CIKey, so have the added cost of lower-casing the key to do the equality check + final TimedCase noCiKeyCase = TimedCase.of("No CIKey", (round, iterations) -> { + doWorkOnThreads(cpuCount, iterations, executorService, () -> { + for (int j = 0; j < keys.size(); j++) { + String key = keys.get(j); + String lowerKey = lowerKeys.get(j); + if (key == null && lowerKey == null) { + // this is ok + } else if (key == null) { + throw new RuntimeException("Mismatch"); + } else { + // In a cache key situation we would have to ensure both sides are lower-cased + key = key.toLowerCase(); + if (!key.equals(lowerKey)) { + throw new RuntimeException("Mismatch"); + } + } + } + }); + }); + + TestUtil.comparePerformance( + 3, + 1_000_000, + LOGGER::info, + commonKeyCase, + dynamicKeyCase, + noCiKeyCase); + } + + private void doWorkOnThreads(final int threadCount, + final long iterations, + final ExecutorService executorService, + final Runnable work) { + + final List> futures = new ArrayList<>(); + for (int i = 0; i < threadCount; i++) { + futures.add(CompletableFuture.runAsync(() -> { + for (int j = 0; j < iterations; j++) { + work.run(); + } + }, executorService)); + } + + for (final CompletableFuture future : futures) { + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIKeyPerf.java b/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIKeyPerf.java new file mode 100644 index 00000000000..d64c1919b77 --- /dev/null +++ b/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIKeyPerf.java @@ -0,0 +1,78 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.shared.string; + +import stroom.util.logging.LambdaLogger; +import stroom.util.logging.LambdaLoggerFactory; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.infra.Blackhole; + +public class TestCIKeyPerf { + + private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(TestCIKeyPerf.class); + + private static final int FORK_COUNT = 1; + private static final int FORK_WARMUPS = 1; + private static final int ITERATIONS = 1; + private static final int THREADS = 10; + + @Threads(THREADS) + @Fork(value = FORK_COUNT, warmups = FORK_WARMUPS) + @Benchmark + @BenchmarkMode(Mode.Throughput) + @Measurement(iterations = ITERATIONS) + public void getUnknownCIKey(Blackhole blackhole) { + CIKey ciKey = CIKey.of("Food"); + blackhole.consume(ciKey); + } + + @Threads(THREADS) + @Fork(value = FORK_COUNT, warmups = FORK_WARMUPS) + @Benchmark + @BenchmarkMode(Mode.Throughput) + @Measurement(iterations = ITERATIONS) + public void getKnownCIKey(Blackhole blackhole) { + CIKey ciKey = CIKey.of("Feed"); + blackhole.consume(ciKey); + } + + @Threads(THREADS) + @Fork(value = FORK_COUNT, warmups = FORK_WARMUPS) + @Benchmark + @BenchmarkMode(Mode.Throughput) + @Measurement(iterations = ITERATIONS) + public void getDynamicCIKey(Blackhole blackhole) { + CIKey ciKey = CIKey.ofDynamicKey("Fume"); + blackhole.consume(ciKey); + } + + @Threads(THREADS) + @Fork(value = FORK_COUNT, warmups = FORK_WARMUPS) + @Benchmark + @BenchmarkMode(Mode.Throughput) + @Measurement(iterations = ITERATIONS) + public void getUnknownLowerCIKey(Blackhole blackhole) { + CIKey ciKey = CIKey.ofLowerCase("feet"); + blackhole.consume(ciKey); + } +} diff --git a/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIKeys.java b/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIKeys.java new file mode 100644 index 00000000000..fe24d8306e4 --- /dev/null +++ b/stroom-util-shared/src/test/java/stroom/util/shared/string/TestCIKeys.java @@ -0,0 +1,69 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.shared.string; + +import stroom.util.logging.LambdaLogger; +import stroom.util.logging.LambdaLoggerFactory; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +class TestCIKeys { + + private static final LambdaLogger LOGGER = LambdaLoggerFactory.getLogger(TestCIKeys.class); + + @Test + void test() { + // ACCEPT is a common key, so any way we get/create it should give us the same string instances. + final CIKey key1 = CIKeys.ACCEPT; + final CIKey key2 = CIKey.of(key1.get()); + final CIKey key3 = CIKey.ofLowerCase(key1.getAsLowerCase()); + final CIKey key4 = CIKey.of("accept"); + // Explicitly don't try to get a common key + final CIKey dynamicKey1 = CIKey.ofDynamicKey("accept"); + final CIKey dynamicKey2 = CIKey.ofDynamicKey("accept"); + + Assertions.assertThat(key1) + .isSameAs(key2); + Assertions.assertThat(key1) + .isSameAs(key3); + Assertions.assertThat(key1) + .isSameAs(key4); + + // Different instances + Assertions.assertThat(key1) + .isNotSameAs(dynamicKey1); + Assertions.assertThat(key1) + .isNotSameAs(dynamicKey2); + Assertions.assertThat(dynamicKey1) + .isNotSameAs(dynamicKey2); + + Assertions.assertThat(key1.get()) + .isSameAs(key2.get()); + Assertions.assertThat(key1.get()) + .isSameAs(key3.get()); + Assertions.assertThat(key1.get()) + .isSameAs(key4.get()); + + Assertions.assertThat(key1.getAsLowerCase()) + .isSameAs(key2.getAsLowerCase()); + Assertions.assertThat(key1.getAsLowerCase()) + .isSameAs(key3.getAsLowerCase()); + Assertions.assertThat(key1.getAsLowerCase()) + .isSameAs(key4.getAsLowerCase()); + } +} diff --git a/stroom-util/src/main/java/stroom/util/AbstractCommandLineTool.java b/stroom-util/src/main/java/stroom/util/AbstractCommandLineTool.java index ef2780ae19b..b4e3730b5e8 100644 --- a/stroom-util/src/main/java/stroom/util/AbstractCommandLineTool.java +++ b/stroom-util/src/main/java/stroom/util/AbstractCommandLineTool.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,8 @@ package stroom.util; +import stroom.util.shared.string.CIKey; + import com.google.common.base.Strings; import java.beans.BeanInfo; @@ -36,7 +38,7 @@ */ public abstract class AbstractCommandLineTool { - private Map map; + private Map map; private List validArguments; private int maxPropLength = 0; @@ -66,7 +68,7 @@ public void init(final String[] args) { if (field.getName().length() > maxPropLength) { maxPropLength = field.getName().length(); } - if (map.containsKey(field.getName())) { + if (map.containsKey(CIKey.of(field.getName()))) { validArguments.add(field.getName()); field.getWriteMethod().invoke(this, getAsType(field)); } @@ -82,16 +84,16 @@ public void init(final String[] args) { private Object getAsType(final PropertyDescriptor descriptor) { final Class propertyClass = descriptor.getPropertyType(); if (propertyClass.equals(String.class)) { - return map.get(descriptor.getName()); + return map.get(CIKey.of(descriptor.getName())); } if (propertyClass.equals(Boolean.class) || propertyClass.equals(Boolean.TYPE)) { - return Boolean.parseBoolean(map.get(descriptor.getName())); + return Boolean.parseBoolean(map.get(CIKey.of(descriptor.getName()))); } if (propertyClass.equals(Integer.class) || propertyClass.equals(Integer.TYPE)) { - return Integer.parseInt(map.get(descriptor.getName())); + return Integer.parseInt(map.get(CIKey.of(descriptor.getName()))); } if (propertyClass.equals(Long.class) || propertyClass.equals(Long.TYPE)) { - return Long.parseLong(map.get(descriptor.getName())); + return Long.parseLong(map.get(CIKey.of(descriptor.getName()))); } throw new RuntimeException("AbstractCommandLineTool does not know about properties of type " + propertyClass); } @@ -117,7 +119,7 @@ private void doTraceArguments(final PrintStream printStream) { if (pd.getWriteMethod() != null) { // Simple getter ? String suffix = " (default)"; - if (map.containsKey(pd.getName())) { + if (map.containsKey(CIKey.of(pd.getName()))) { suffix = " (arg)"; } String value = ""; diff --git a/stroom-util/src/main/java/stroom/util/ArgsUtil.java b/stroom-util/src/main/java/stroom/util/ArgsUtil.java index 3804a8fe19b..b7ba8be379f 100644 --- a/stroom-util/src/main/java/stroom/util/ArgsUtil.java +++ b/stroom-util/src/main/java/stroom/util/ArgsUtil.java @@ -1,5 +1,25 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.util; +import stroom.util.shared.string.CIKey; + +import java.util.Collections; +import java.util.HashMap; import java.util.Map; public final class ArgsUtil { @@ -8,16 +28,21 @@ private ArgsUtil() { // Utility class. } - public static Map parse(final String[] args) { - final CIStringHashMap map = new CIStringHashMap(); - for (final String arg : args) { - final String[] split = arg.split("="); - if (split.length > 1) { - map.put(split[0], split[1]); - } else { - map.put(split[0], ""); + public static Map parse(final String[] args) { + if (NullSafe.isEmptyArray(args)) { + return Collections.emptyMap(); + } else { + final HashMap map = new HashMap<>(args.length); + for (final String arg : args) { + final String[] split = arg.split("="); + CIKey key = CIKey.of(split[0]); + if (split.length > 1) { + map.put(key, split[1]); + } else { + map.put(key, ""); + } } + return map; } - return map; } } diff --git a/stroom-util/src/main/java/stroom/util/NullSafe.java b/stroom-util/src/main/java/stroom/util/NullSafe.java index 8692f2fb591..be7b833d9c6 100644 --- a/stroom-util/src/main/java/stroom/util/NullSafe.java +++ b/stroom-util/src/main/java/stroom/util/NullSafe.java @@ -50,19 +50,38 @@ private NullSafe() { /** * Allows you to safely compare a child property of val1 to other. * - * @return False if val1 is null else whether the child property of val1 is equal to other + * @return Whether the child property of val1 is equal to other or + * if both val1 and other are null. */ public static boolean equals(final T1 val1, final Function getter, final Object other) { if (val1 == null) { - return false; + return other == null; } else { final T2 val2 = getter.apply(val1); return Objects.equals(val2, other); } } + /** + * Allows you to safely compare a child property of val1 to other. + * + * @return Whether the child property of val1 is equal to other or + * if both val1 and other are null. + */ + public static boolean equalsIgnoreCase(final T val1, + final Function getter, + final String other) { + if (val1 == null) { + return other == null; + } else { + final String val2 = getter.apply(val1); + return val2 != null + && val2.equalsIgnoreCase(other); + } + } + /** * Test if the properties (accessed using the same getter for both) of two * objects of the same class are equal in a null safe way. @@ -163,25 +182,37 @@ public static Optional firstNonNull(final T... vals) { } +// /** +// * @return The first item in the list or null if list is null or empty. +// */ +// public static T first(final List list) { +// if (list == null || list.isEmpty()) { +// return null; +// } else { +// return list.getFirst(); +// } +// } + /** * @return The first item in the list or null if list is null or empty. */ - public static T first(final List list) { + public static T last(final List list) { if (list == null || list.isEmpty()) { return null; } else { - return list.getFirst(); + return list.getLast(); } } /** - * @return The first item in the list or null if list is null or empty. + * Return first value in the list or an empty {@link Optional} if the list + * is null, empty or the first item is null. */ - public static T last(final List list) { + public static Optional first(final List list) { if (list == null || list.isEmpty()) { - return null; + return Optional.empty(); } else { - return list.getLast(); + return Optional.ofNullable(list.getFirst()); } } diff --git a/stroom-util/src/main/java/stroom/util/PredicateUtil.java b/stroom-util/src/main/java/stroom/util/PredicateUtil.java index d6deeb510fe..3115cd2000d 100644 --- a/stroom-util/src/main/java/stroom/util/PredicateUtil.java +++ b/stroom-util/src/main/java/stroom/util/PredicateUtil.java @@ -1,3 +1,19 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package stroom.util; import stroom.util.string.PatternUtil; @@ -13,6 +29,7 @@ public class PredicateUtil { private static final String LIST_DELIMITER = ","; private static final Pattern LIST_DELIMITER_PATTERN = Pattern.compile(LIST_DELIMITER); + public static final Predicate ALWAYS_TRUE_STRING_PREDICATE = val -> true; private PredicateUtil() { // Static utils only @@ -73,6 +90,7 @@ public static Predicate createWildCardedFilterPredicate(final String fil /** * Create a predicate ORing all the filter terms in inListStr together. Supports standard stroom * wild carding with '*'. Matches are complete matches. + * * @param inListStr List of filter terms delimited by LIST_DELIMITER */ public static Predicate createWildCardedInPredicate(final String inListStr, diff --git a/stroom-util/src/main/java/stroom/util/string/MultiCaseMap.java b/stroom-util/src/main/java/stroom/util/string/MultiCaseMap.java new file mode 100644 index 00000000000..4025dd7607b --- /dev/null +++ b/stroom-util/src/main/java/stroom/util/string/MultiCaseMap.java @@ -0,0 +1,192 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.string; + +import stroom.util.NullSafe; +import stroom.util.shared.GwtNullSafe; +import stroom.util.shared.string.CIKey; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +/** + * Wraps a nested map that allows you to perform both case-sensitive and + * case-insensitive look-ups. + */ +public class MultiCaseMap { + + @SuppressWarnings("rawtypes") + private static final MultiCaseMap EMPTY = new MultiCaseMap<>(Collections.emptyMap()); + + private final Map> map; + + public MultiCaseMap() { + this.map = new HashMap<>(); + } + + private MultiCaseMap(final Map> map) { + this.map = map; + } + + public static MultiCaseMap fromStringKeyedMap(final Map map) { + final MultiCaseMap multiCaseMap = new MultiCaseMap<>(); + NullSafe.map(map) + .forEach(multiCaseMap::put); + return multiCaseMap; + } + + @SuppressWarnings("unchecked") + public static MultiCaseMap emptyMap() { + return (MultiCaseMap) EMPTY; + } + + /** + * Case-sensitive match on key + */ + public boolean containsKey(final String key) { + return GwtNullSafe.test( + map.get(CIKey.of(key)), + subMap -> subMap.containsKey(key)); + } + + /** + * Case-insensitive match on ciKey + */ + public boolean containsKeys(final CIKey key) { + final Map subMap = map.get(key); + return subMap != null && !subMap.isEmpty(); + } + + public int size() { + return map.size(); + } + + public boolean isEmpty() { + return map.isEmpty(); + } + + public void clear() { + map.clear(); + } + + /** + * Case-sensitive match on key + */ + public T get(final String key) { + return GwtNullSafe.get( + map.get(CIKey.of(key)), + subMap -> subMap.get(key)); + } + + /** + * Case-insensitive match on {@link CIKey}. + * + * @return A map of values keyed on a case-sensitive key. + */ + public Map get(final CIKey key) { + return GwtNullSafe.requireNonNullElseGet(map.get(key), Collections::emptyMap); + } + + /** + * @return The value associated with ciKey (case-insensitive) or null if there + * is no associated value. If ciKey matches multiple keys with different case then + * it will attempt to do a case-sensitive match. If there is no case-sensitive match + * it will throw a {@link MultipleMatchException}. + * @throws MultipleMatchException if multiple values are associated with ciKey + */ + public T getCaseSensitive(final CIKey ciKey) { + final Map subMap = map.get(ciKey); + if (NullSafe.isEmptyMap(subMap)) { + return null; + } else { + final int count = subMap.size(); + if (count == 0) { + // Should never happen. Shouldn't have an empty subMap + throw new RuntimeException("Empty subMap"); + } else if (count == 1) { + // There is only one, so we don't need to check for an exact match + return subMap.values().iterator().next(); + } else { + final T exactMatch = subMap.get(ciKey.get()); + if (exactMatch != null) { + return exactMatch; + } else { + throw new MultipleMatchException("Multiple values (" + subMap.size() + + ") exist for case-insensitive key '" + ciKey.get() + "'"); + } + } + } + } + + /** + * Put an entry into the map + */ + public T put(final String key, final T value) { + return map.computeIfAbsent(CIKey.of(key), k -> new HashMap<>()) + .put(key, value); + } + + /** + * Put an entry into the map + */ + public T put(final CIKey ciKey, final T value) { + return map.computeIfAbsent(ciKey, k -> new HashMap<>()) + .put(ciKey.get(), value); + } + + /** + * Iterate over case-sensitive entry keys + */ + public void forEach(final BiConsumer action) { + if (action != null) { + map.values() + .stream() + .filter(Objects::nonNull) + .flatMap(subMap -> + subMap.entrySet() + .stream()) + .forEach(entry -> action.accept(entry.getKey(), entry.getValue())); + } + } + + public Set> entrySet() { + return map.values() + .stream() + .filter(Objects::nonNull) + .flatMap(subMap -> + subMap.entrySet() + .stream()) + .collect(Collectors.toSet()); + } + + + // -------------------------------------------------------------------------------- + + + public static class MultipleMatchException extends RuntimeException { + + public MultipleMatchException(final String message) { + super(message); + } + } +} diff --git a/stroom-util/src/test/java/stroom/util/TestNullSafe.java b/stroom-util/src/test/java/stroom/util/TestNullSafe.java index 3ee61d6ec6c..29e7535096f 100644 --- a/stroom-util/src/test/java/stroom/util/TestNullSafe.java +++ b/stroom-util/src/test/java/stroom/util/TestNullSafe.java @@ -90,8 +90,34 @@ void testEquals1() { Level1::getNonNullLevel2, nonNullLevel1.getNonNullLevel2())) .isTrue(); + assertThat(NullSafe.equals(null, Level1::getNonNullLevel2, null)) + .isTrue(); } + @TestFactory + Stream testEqualsIgnoreCase1() { + return TestUtil.buildDynamicTestStream() + .withWrappedInputType(new TypeLiteral, String>>() { + }) + .withOutputType(boolean.class) + .withTestFunction(testCase -> { + final AtomicReference ref = testCase.getInput()._1; + final String other = testCase.getInput()._2; + return NullSafe.equalsIgnoreCase(ref, AtomicReference::get, other); + }) + .withSimpleEqualityAssertion() + .addCase(Tuple.of(null, "foo"), false) + .addCase(Tuple.of(new AtomicReference<>(), "foo"), false) + .addCase(Tuple.of(new AtomicReference<>("bar"), "foo"), false) + .addCase(Tuple.of(new AtomicReference<>("foo"), "foo"), true) + .addCase(Tuple.of(new AtomicReference<>("FOO"), "foo"), true) + .addCase(Tuple.of(new AtomicReference<>("foo"), "FOO"), true) + .addCase(Tuple.of(null, null), true) + .addCase(Tuple.of(new AtomicReference<>(), null), false) + .build(); + } + + @Test void testEquals2() { // Null parent @@ -219,14 +245,16 @@ Stream testFirst() { return TestUtil.buildDynamicTestStream() .withWrappedInputType(new TypeLiteral>() { }) - .withOutputType(String.class) - .withTestFunction(testCase -> - NullSafe.first(testCase.getInput())) + .withWrappedOutputType(new TypeLiteral>() { + }) + .withSingleArgTestFunction(NullSafe::first) .withSimpleEqualityAssertion() - .addCase(null, null) - .addCase(Collections.emptyList(), null) - .addCase(List.of("bar"), "bar") - .addCase(List.of("foo", "bar"), "foo") + .addCase(null, Optional.empty()) + .addCase(Collections.emptyList(), Optional.empty()) + .addCase(Arrays.asList((String) null), Optional.empty()) + .addCase(Arrays.asList((String) null, (String) null), Optional.empty()) + .addCase(List.of("foo"), Optional.of("foo")) + .addCase(List.of("foo", "bar"), Optional.of("foo")) .build(); } diff --git a/stroom-util/src/test/java/stroom/util/string/TestMultiCaseMap.java b/stroom-util/src/test/java/stroom/util/string/TestMultiCaseMap.java new file mode 100644 index 00000000000..b195032b2f9 --- /dev/null +++ b/stroom-util/src/test/java/stroom/util/string/TestMultiCaseMap.java @@ -0,0 +1,108 @@ +/* + * Copyright 2024 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package stroom.util.string; + +import stroom.util.string.MultiCaseMap.MultipleMatchException; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static stroom.util.shared.string.CIKey.of; + +class TestMultiCaseMap { + + @Test + void containsKey() { + final MultiCaseMap map = new MultiCaseMap<>(); + map.put("a", "a1"); + map.put("A", "a2"); + + map.put("b", "b1"); + + assertThat(map.containsKey("a")) + .isTrue(); + assertThat(map.containsKey("A")) + .isTrue(); + assertThat(map.containsKey("b")) + .isTrue(); + assertThat(map.containsKey("B")) + .isFalse(); + } + + @Test + void entrySet() { + final MultiCaseMap map = new MultiCaseMap<>(); + map.put("a", "a1"); + map.put("A", "a2"); + + map.put("b", "b1"); + + assertThat(map.entrySet()) + .containsExactlyInAnyOrder( + Map.entry("a", "a1"), + Map.entry("A", "a2"), + Map.entry("b", "b1")); + } + + @Test + void getCaseSensitive() { + final MultiCaseMap map = new MultiCaseMap<>(); + map.put("a", "a1"); + map.put("A", "a2"); + + map.put("b", "b1"); + + assertThat(map.get(of("a")).keySet()) + .containsExactlyInAnyOrder("a", "A"); + assertThat(map.get(of("A")).keySet()) + .containsExactlyInAnyOrder("a", "A"); + assertThat(map.getCaseSensitive(of("a"))) + .isEqualTo("a1"); + assertThat(map.getCaseSensitive(of("A"))) + .isEqualTo("a2"); + + assertThat(map.get(of("b")).keySet()) + .containsExactlyInAnyOrder("b"); + assertThat(map.get(of("B")).keySet()) + .containsExactlyInAnyOrder("b"); + assertThat(map.getCaseSensitive(of("b"))) + .isEqualTo("b1"); + assertThat(map.getCaseSensitive(of("B"))) + .isEqualTo("b1"); + } + + @Test + void getCaseSensitive_throws() { + + final MultiCaseMap map = new MultiCaseMap<>(); + map.put("foo", "f1"); + map.put("FOO", "f2"); + map.put("Foo", "f3"); + + assertThat(map.get(of("foO")).keySet()) + .containsExactlyInAnyOrder("foo", "FOO", "Foo"); + + Assertions.assertThatThrownBy( + () -> { + map.getCaseSensitive(of("fOO")); + }) + .isInstanceOf(MultipleMatchException.class); + } +} diff --git a/stroom-util/src/test/java/stroom/util/testshared/TestCompareUtil.java b/stroom-util/src/test/java/stroom/util/testshared/TestCompareUtil.java index c7947568ef4..9e42a41a4e1 100644 --- a/stroom-util/src/test/java/stroom/util/testshared/TestCompareUtil.java +++ b/stroom-util/src/test/java/stroom/util/testshared/TestCompareUtil.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 Crown Copyright + * Copyright 2024 Crown Copyright * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,23 +17,29 @@ package stroom.util.testshared; +import stroom.util.NullSafe; import stroom.util.shared.CompareUtil; import org.junit.jupiter.api.Test; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; + import static org.assertj.core.api.Assertions.assertThat; class TestCompareUtil { @Test void testStringCompare() { - assertThat(CompareUtil.compareString(null, null)).isEqualTo(0); - assertThat(CompareUtil.compareString("A", "A")).isEqualTo(0); - assertThat(CompareUtil.compareString("A", "a")).isEqualTo(0); - assertThat(CompareUtil.compareString("A", "B")).isEqualTo(-1); - assertThat(CompareUtil.compareString("B", "a")).isEqualTo(1); - assertThat(CompareUtil.compareString("B", null)).isEqualTo(1); - assertThat(CompareUtil.compareString(null, "B")).isEqualTo(-1); + assertThat(CompareUtil.compareStringIgnoreCase(null, null)).isEqualTo(0); + assertThat(CompareUtil.compareStringIgnoreCase("A", "A")).isEqualTo(0); + assertThat(CompareUtil.compareStringIgnoreCase("A", "a")).isEqualTo(0); + assertThat(CompareUtil.compareStringIgnoreCase("A", "B")).isEqualTo(-1); + assertThat(CompareUtil.compareStringIgnoreCase("B", "a")).isEqualTo(1); + assertThat(CompareUtil.compareStringIgnoreCase("B", null)).isEqualTo(1); + assertThat(CompareUtil.compareStringIgnoreCase(null, "B")).isEqualTo(-1); } @Test @@ -46,4 +52,60 @@ void testLongCompare() { assertThat(CompareUtil.compareLong(null, 2L)).isEqualTo(-1); } + @Test + void testGetNullSafeCaseInsensitiveComparator() { + + Comparator> comparator = CompareUtil.getNullSafeCaseInsensitiveComparator( + AtomicReference::get); + + final List> list = Stream.of( + "A", + "b", + "a", + "c", + null, + "C", + "2", + "1", + "0") + .map(AtomicReference::new) + .sorted(comparator) + .toList(); + + assertThat(list) + .extracting(AtomicReference::get) + .containsExactly( + null, + "0", + "1", + "2", + "A", + "a", + "b", + "c", + "C"); + } + + @Test + void testGetNullSafeCaseInsensitiveComparator2() { + + Comparator> comparator = CompareUtil.getNullSafeCaseInsensitiveComparator( + AtomicReference::get); + + final List> list = Stream.of( + new AtomicReference<>("b"), + new AtomicReference<>("A"), + new AtomicReference<>((String) null), + null) + .sorted(comparator) + .toList(); + + assertThat(list) + .extracting(ref -> NullSafe.get(ref, AtomicReference::get)) + .containsExactly( + null, + null, + "A", + "b"); + } } diff --git a/stroom-view/stroom-view-impl/src/main/java/stroom/view/impl/ViewSearchProvider.java b/stroom-view/stroom-view-impl/src/main/java/stroom/view/impl/ViewSearchProvider.java index 79e5cb3b4c8..c18a67711c7 100644 --- a/stroom-view/stroom-view-impl/src/main/java/stroom/view/impl/ViewSearchProvider.java +++ b/stroom-view/stroom-view-impl/src/main/java/stroom/view/impl/ViewSearchProvider.java @@ -18,7 +18,6 @@ import stroom.datasource.api.v2.DataSourceProvider; import stroom.datasource.api.v2.FindFieldCriteria; -import stroom.datasource.api.v2.IndexField; import stroom.datasource.api.v2.QueryField; import stroom.docref.DocRef; import stroom.docstore.shared.DocRefUtil; @@ -27,6 +26,7 @@ import stroom.query.api.v2.SearchRequest; import stroom.query.api.v2.TableSettings; import stroom.query.common.v2.DataSourceProviderRegistry; +import stroom.query.common.v2.IndexFieldMap; import stroom.query.common.v2.IndexFieldProvider; import stroom.query.common.v2.IndexFieldProviders; import stroom.query.common.v2.ResultStore; @@ -36,6 +36,7 @@ import stroom.util.logging.LambdaLogger; import stroom.util.logging.LambdaLoggerFactory; import stroom.util.shared.ResultPage; +import stroom.util.shared.string.CIKey; import stroom.view.api.ViewStore; import stroom.view.shared.ViewDoc; @@ -121,10 +122,10 @@ public int getFieldCount(final DocRef viewDocRef) { } @Override - public IndexField getIndexField(final DocRef viewDocRef, final String fieldName) { + public IndexFieldMap getIndexFields(final DocRef viewDocRef, final CIKey fieldName) { final DocRef docRef = getReferencedDataSource(viewDocRef); if (docRef != null) { - return indexFieldProviders.getIndexField(docRef, fieldName); + return indexFieldProviders.getIndexFields(docRef, fieldName); } return null; } diff --git a/stroom-view/stroom-view-impl/src/main/java/stroom/view/impl/ViewStoreImpl.java b/stroom-view/stroom-view-impl/src/main/java/stroom/view/impl/ViewStoreImpl.java index 8eb61ab092e..c553d1173af 100644 --- a/stroom-view/stroom-view-impl/src/main/java/stroom/view/impl/ViewStoreImpl.java +++ b/stroom-view/stroom-view-impl/src/main/java/stroom/view/impl/ViewStoreImpl.java @@ -217,8 +217,10 @@ public List list() { } @Override - public List findByNames(final List name, final boolean allowWildCards) { - return store.findByNames(name, allowWildCards); + public List findByNames(final List names, + final boolean allowWildCards, + final boolean isCaseSensitive) { + return store.findByNames(names, allowWildCards, isCaseSensitive); } @Override diff --git a/unreleased_changes/20240726_161851_024__4365.md b/unreleased_changes/20240726_161851_024__4365.md new file mode 100644 index 00000000000..72c9cdc7515 --- /dev/null +++ b/unreleased_changes/20240726_161851_024__4365.md @@ -0,0 +1,25 @@ +* Issue **#4365** : Change stroomQL to be case-insensitive in terms of fields, eval variables, data source names, visualisation names, visualisation param names. Change the way stroom works with fields from index providers (e.g. Lucene, Elastic, etc.) so that stroom ignores case unless it finds multiple matching field names in which case it will try a case-sensitive match. If there is no case-sensitive match in this scenario then the query will error. + + + +```sh +# ******************************************************************************** +# Issue title: StroomQL is accepting case-insensitive Field names +# Issue link: https://github.com/gchq/stroom/issues/4365 +# ******************************************************************************** + +# ONLY the top line will be included as a change entry in the CHANGELOG. +# The entry should be in GitHub flavour markdown and should be written on a SINGLE +# line with no hard breaks. You can have multiple change files for a single GitHub issue. +# The entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than +# 'Fixed nasty bug'. +# +# Examples of acceptable entries are: +# +# +# * Issue **123** : Fix bug with an associated GitHub issue in this repository +# +# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository +# +# * Fix bug with no associated GitHub issue. +``` diff --git a/unreleased_changes/20240829_163948_539__0.md b/unreleased_changes/20240829_163948_539__0.md new file mode 100644 index 00000000000..8a7b9a8cb85 --- /dev/null +++ b/unreleased_changes/20240829_163948_539__0.md @@ -0,0 +1,19 @@ +* Refactor the way the case-insensitive meta attributes are held in memory. There is no change to functionality. + + +```sh +# ONLY the top line will be included as a change entry in the CHANGELOG. +# The entry should be in GitHub flavour markdown and should be written on a SINGLE +# line with no hard breaks. You can have multiple change files for a single GitHub issue. +# The entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than +# 'Fixed nasty bug'. +# +# Examples of acceptable entries are: +# +# +# * Issue **123** : Fix bug with an associated GitHub issue in this repository +# +# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository +# +# * Fix bug with no associated GitHub issue. +``` diff --git a/unreleased_changes/20240912_144431_092__0.md b/unreleased_changes/20240912_144431_092__0.md new file mode 100644 index 00000000000..4f760b20a1d --- /dev/null +++ b/unreleased_changes/20240912_144431_092__0.md @@ -0,0 +1,19 @@ +* Add the `Rows` searchable which returns 100k rows with `RowNum` values of 1->100k. + + +```sh +# ONLY the top line will be included as a change entry in the CHANGELOG. +# The entry should be in GitHub flavour markdown and should be written on a SINGLE +# line with no hard breaks. You can have multiple change files for a single GitHub issue. +# The entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than +# 'Fixed nasty bug'. +# +# Examples of acceptable entries are: +# +# +# * Issue **123** : Fix bug with an associated GitHub issue in this repository +# +# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository +# +# * Fix bug with no associated GitHub issue. +```