Skip to content

Ioc upload integ tests and fix update (#1162) #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: 2.13-threat-intel
Choose a base branch
from
Open

Ioc upload integ tests and fix update (#1162) #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions src/main/java/org/opensearch/securityanalytics/services/STIX2IOCFeedStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,15 +114,17 @@ public void indexIocs(List<STIX2IOC> iocs) throws IOException {

initFeedIndex(newActiveIndex, ActionListener.wrap(
r -> {
// reset the store configs
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().clear();
}

// recreate the store configs
saTifSourceConfig.getIocTypes().forEach(type -> {
IOCType iocType = new IOCType(type);
if (saTifSourceConfig.getIocStoreConfig() instanceof DefaultIocStoreConfig) {
List<DefaultIocStoreConfig.IocToIndexDetails> listOfIocToIndexDetails =
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails();
listOfIocToIndexDetails.removeIf(iocToIndexDetails -> iocToIndexDetails.getIocType() == iocType);
DefaultIocStoreConfig.IocToIndexDetails iocToIndexDetails =
new DefaultIocStoreConfig.IocToIndexDetails(iocType, iocIndexPattern, newActiveIndex);
listOfIocToIndexDetails.add(iocToIndexDetails);
new DefaultIocStoreConfig.IocToIndexDetails(new IOCType(type), iocIndexPattern, newActiveIndex);
((DefaultIocStoreConfig) saTifSourceConfig.getIocStoreConfig()).getIocToIndexDetails().add(iocToIndexDetails);
}
});
bulkIndexIocs(iocs, newActiveIndex);
Expand Down
55 changes: 34 additions & 21 deletions .../opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,15 @@
import org.opensearch.securityanalytics.threatIntel.model.IocUploadSource;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig;
import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;

import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
Expand Down Expand Up @@ -158,7 +160,7 @@ public void createIocAndTIFSourceConfig(
));
},
e -> {
log.error("Failed to download and save IOCs for source config [{}]", indexSaTifSourceConfigResponse.getId());
log.error("Failed to download and save IOCs for threat intel source config [{}]", indexSaTifSourceConfigResponse.getId());
saTifSourceConfigService.deleteTIFSourceConfig(indexSaTifSourceConfigResponse, ActionListener.wrap(
deleteResponse -> {
log.debug("Successfully deleted threat intel source config [{}]", indexSaTifSourceConfigResponse.getId());
Expand Down Expand Up @@ -204,15 +206,15 @@ public void downloadAndSaveIOCs(SATIFSourceConfig saTifSourceConfig,
if (saTifSourceConfig.getIocTypes().contains(stix2IOC.getType().toString())) {
validStix2IocList.add(stix2IOC);
} else {
log.error("{} is not a supported Ioc type for tif source config {}. Skipping IOC {}: of type {} value {}",
log.error("{} is not a supported Ioc type for threat intel source config {}. Skipping IOC {}: of type {} value {}",
stix2IOC.getType().toString(), saTifSourceConfig.getId(),
stix2IOC.getId(), stix2IOC.getType().toString(), stix2IOC.getValue()
);
}
}
if (validStix2IocList.isEmpty()) {
log.error("No supported IOCs to index");
actionListener.onFailure(new OpenSearchStatusException("No compatible Iocs were uploaded for config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST));
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("No compatible Iocs were uploaded for threat intel source config " + saTifSourceConfig.getName(), RestStatus.BAD_REQUEST)));
return;
}
stix2IOCFetchService.onlyIndexIocs(saTifSourceConfig, validStix2IocList, actionListener);
Expand Down Expand Up @@ -274,14 +276,18 @@ public void updateIocAndTIFSourceConfig(
saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigDto.getId(), ActionListener.wrap(
retrievedSaTifSourceConfig -> {
if (TIFJobState.AVAILABLE.equals(retrievedSaTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(retrievedSaTifSourceConfig.getState()) == false) {
log.error("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState());
listener.onFailure(new OpenSearchException("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()));
log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState());
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
String.format(Locale.getDefault(), "Invalid threat intel source config state. Expecting %s or %s but received %s", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, retrievedSaTifSourceConfig.getState()),
RestStatus.BAD_REQUEST)));
return;
}

if (false == saTifSourceConfigDto.getType().equals(retrievedSaTifSourceConfig.getType())) {
log.error("Unable to update source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType());
listener.onFailure(new OpenSearchException("Unable to update source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()));
log.error("Unable to update threat intel source config, type cannot change from {} to {}", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType());
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
String.format(Locale.getDefault(), "Unable to update threat intel source config, type cannot change from %s to %s", retrievedSaTifSourceConfig.getType(), saTifSourceConfigDto.getType()),
RestStatus.BAD_REQUEST)));
return;
}

Expand Down Expand Up @@ -345,7 +351,6 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene

saTifSourceConfigService.getClusterState(ActionListener.wrap(
clusterStateResponse -> {
List<String> iocTypes = updatedSaTifSourceConfig.getIocTypes();
IocStoreConfig iocStoreConfig = updatedSaTifSourceConfig.getIocStoreConfig();
Set<String> activeIndices = new HashSet<>();
Set<String> indicesToDelete = new HashSet<>();
Expand Down Expand Up @@ -388,11 +393,13 @@ private void storeAndDeleteIocIndices(List<STIX2IOC> stix2IOCList, ActionListene
), iocIndexPatterns.toArray(new String[0]));
},
e -> {
log.error("Failed to download and save IOCs for source config [{}]", updatedSaTifSourceConfig.getId());
log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSaTifSourceConfig.getId(), e);
markSourceConfigAsAction(updatedSaTifSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap(
r -> {
log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId());
listener.onFailure(new OpenSearchException("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId()));
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException(
String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSaTifSourceConfig.getId()),
e)));
}, ex -> {
log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSaTifSourceConfig.getId());
listener.onFailure(ex);
Expand Down Expand Up @@ -423,14 +430,18 @@ public void refreshTIFSourceConfig(
saTifSourceConfigService.getTIFSourceConfig(saTifSourceConfigId, ActionListener.wrap(
saTifSourceConfig -> {
if (saTifSourceConfig.getType() == IOC_UPLOAD) {
log.error("Unable to refresh source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD);
listener.onFailure(new OpenSearchException("Unable to refresh source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD));
log.error("Unable to refresh threat intel source config [{}] with a source type of [{}]", saTifSourceConfig.getId(), IOC_UPLOAD);
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
String.format(Locale.getDefault(), "Unable to refresh threat intel source config [%s] with a source type of [%s]", saTifSourceConfig.getId(), IOC_UPLOAD),
RestStatus.BAD_REQUEST)));
return;
}

if (TIFJobState.AVAILABLE.equals(saTifSourceConfig.getState()) == false && TIFJobState.REFRESH_FAILED.equals(saTifSourceConfig.getState()) == false) {
log.error("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState());
listener.onFailure(new OpenSearchException("Invalid TIF job state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()));
log.error("Invalid threat intel source config state. Expecting {} or {} but received {}", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState());
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(
String.format(Locale.getDefault(), "Invalid threat intel source config state. Expecting %s or %s but received %s", TIFJobState.AVAILABLE, TIFJobState.REFRESH_FAILED, saTifSourceConfig.getState()),
RestStatus.BAD_REQUEST)));
return;
}

Expand Down Expand Up @@ -488,14 +499,16 @@ private void downloadAndSaveIocsToRefresh(ActionListener<SATIFSourceConfigDto> l
));
}, downloadAndSaveIocsError -> {
// Update source config as refresh failed
log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSourceConfig.getId());
log.error("Failed to download and save IOCs for threat intel source config [{}]", updatedSourceConfig.getId(), downloadAndSaveIocsError);
markSourceConfigAsAction(updatedSourceConfig, TIFJobState.REFRESH_FAILED, ActionListener.wrap(
r -> {
log.debug("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId());
listener.onFailure(new OpenSearchException("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId()));
}, e -> {
log.info("Set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId());
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchException(
String.format(Locale.getDefault(), "Failed to download and save IOCs for threat intel source config [%s]. Set source config as REFRESH_FAILED", updatedSourceConfig.getId()),
downloadAndSaveIocsError)));
}, ex -> {
log.error("Failed to set threat intel source config as REFRESH_FAILED for [{}]", updatedSourceConfig.getId());
listener.onFailure(e);
listener.onFailure(ex);
}
));
}));
Expand Down Expand Up @@ -528,7 +541,7 @@ public void deleteTIFSourceConfig(
}, e -> {
log.error("Failed to get threat intel source config for [{}]", saTifSourceConfigId);
if (e instanceof IndexNotFoundException) {
listener.onFailure(new OpenSearchException("Threat intel source config [{}] not found", saTifSourceConfigId));
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", saTifSourceConfigId), RestStatus.NOT_FOUND)));
} else {
listener.onFailure(e);
}
Expand Down Expand Up @@ -708,7 +721,7 @@ private void deleteAllIocsAndSourceConfig(String saTifSourceConfigId, ActionList
}
));
}, e -> {
log.error("Failed to delete IOC indices for source config [{}]", updateSaTifSourceConfigResponse.getId());
log.error("Failed to delete IOC indices for threat intel source config [{}]", updateSaTifSourceConfigResponse.getId());
listener.onFailure(e);
}
));
Expand Down
11 changes: 5 additions & 6 deletions ...n/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.OpenSearchException;
import org.opensearch.OpenSearchStatusException;
import org.opensearch.ResourceAlreadyExistsException;
import org.opensearch.action.StepListener;
Expand Down Expand Up @@ -226,7 +225,7 @@ public void getTIFSourceConfig(
client.get(getRequest, ActionListener.wrap(
getResponse -> {
if (!getResponse.isExists()) {
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config not found.", RestStatus.NOT_FOUND)));
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"Threat intel source config [%s] not found.", tifSourceConfigId), RestStatus.NOT_FOUND)));
return;
}
SATIFSourceConfig saTifSourceConfig = null;
Expand All @@ -238,7 +237,7 @@ public void getTIFSourceConfig(
saTifSourceConfig = SATIFSourceConfig.docParse(xcp, getResponse.getId(), getResponse.getVersion());
}
if (saTifSourceConfig == null) {
actionListener.onFailure(new OpenSearchException("No threat intel source config exists [{}]", tifSourceConfigId));
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(),"No threat intel source config exists [%s]", tifSourceConfigId), RestStatus.BAD_REQUEST)));
} else {
log.debug("Threat intel source config with id [{}] fetched", getResponse.getId());
actionListener.onResponse(saTifSourceConfig);
Expand All @@ -258,7 +257,7 @@ public void searchTIFSourceConfigs(

// Check to make sure the job index exists
if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) {
actionListener.onFailure(new OpenSearchException("Threat intel source config index does not exist"));
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST)));
return;
}

Expand Down Expand Up @@ -350,7 +349,7 @@ public void deleteTIFSourceConfig(
) {
// check to make sure the job index exists
if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == false) {
actionListener.onFailure(new OpenSearchException("Threat intel source config index does not exist"));
actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config index does not exist", RestStatus.BAD_REQUEST)));
return;
}

Expand Down Expand Up @@ -410,7 +409,7 @@ private void deleteIocIndex(Set<String> indicesToDelete, Boolean backgroundJob,
if (!response.isAcknowledged()) {
log.error("Could not delete one or more IOC indices: " + index);
if (backgroundJob == false) {
listener.onFailure(new OpenSearchException("Could not delete one or more IOC indices: " + index));
listener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException(String.format(Locale.getDefault(), "Could not delete one or more IOC indices: " + index), RestStatus.INTERNAL_SERVER_ERROR)));
}
} else {
log.debug("Successfully deleted one or more IOC indices:" + index);
Expand Down
Loading