diff --git a/.travis.yml b/.travis.yml
index 7792b772..5719cbcb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,9 +11,6 @@ cache:
install: true
script:
- set -e
- - echo -en "travis_fold:start:Test\r"
- - mvn install -Pdependency-check -B
- - echo -en "travis_fold:end:Test\r"
- export REPO=securecodebox/engine
- export TAG=$(echo $TRAVIS_BRANCH | sed 's/\//-/g')
- echo -en "travis_fold:start:Docker_Build\r"
diff --git a/README.md b/README.md
index 0d9cc7d0..44935c38 100644
--- a/README.md
+++ b/README.md
@@ -36,6 +36,51 @@ This allows you to e.g. enable https using:
| SERVER_SSL_ENABLED | Enables http over ssl | true |
| SERVER_SSL_KEY_STORE_PASSWORD | Password to the java keystore | AStrongPassword-NotThisOne! |
+## Persistence Provider Configuration
+A more detailed description of all persistence specific integration configuration options can be fund here: [secureCodeBox Integration Documentation][scb-integration]
+
+### Enabling Elasticsearch as Persistence Provider
+All properties defined in scb-engine/src/main/resources/application.yaml can be overwritten via environment variables.
+
+| Property | Example Value |
+| ---------------------------------------------------- | -------------------------- |
+| SECURECODEBOX_PERSISTENCE_ELASTICSEARCH_ENABLED | true |
+| SECURECODEBOX_PERSISTENCE_ELASTICSEARCH_HOST | elasticsearch.example.com |
+| SECURECODEBOX_PERSISTENCE_ELASTICSEARCH_PORT | 9200 |
+| SECURECODEBOX_PERSISTENCE_ELASTICSEARCH_INDEX_PREFIX | securecodebox |
+
+### Configure Elasticsearch Basic Authentication
+If your elasticsearch service enforces authentication your can configure basic authentication:
+
+| Property | Example Value |
+| ----------------------------------------------------------- | --------------------------- |
+| SECURECODEBOX_PERSISTENCE_ELASTICSEARCH_AUTH | basic |
+| SECURECODEBOX_PERSISTENCE_ELASTICSEARCH_AUTH_BASIC_USERNAME | elastic |
+| SECURECODEBOX_PERSISTENCE_ELASTICSEARCH_AUTH_BASIC_PASSWORD | AStrongPassword-NotThisOne! |
+
+### Configure Elasticsearch API Token Authentication
+If your elasticsearch service enforces authentication your can configure api token based authentication:
+
+| Property | Example Value |
+| ----------------------------------------------------------- | --------------------------- |
+| SECURECODEBOX_PERSISTENCE_ELASTICSEARCH_AUTH | token |
+| SECURECODEBOX_PERSISTENCE_ELASTICSEARCH_AUTH_APIKEY_ID | yourToken |
+| SECURECODEBOX_PERSISTENCE_ELASTICSEARCH_AUTH_APIKEY_SECRET | 7fd7eac6fed567b19932492347 |
+
+### Enabling DefectDojo as Persistence Provider
+All properties defined in scb-engine/src/main/resources/application.yaml can be overwritten via environment variables.
+
+#### Properties / Environment Variables
+
+| Property | Example Value |
+| ---------------------------------------------- | ---------------------------------------- |
+| SECURECODEBOX_PERSISTENCE_DEFECTDOJO_ENABLED | true |
+| SECURECODEBOX_PERSISTENCE_DEFECTDOJO_URL | [http://localhost:8000]() |
+| SECURECODEBOX_PERSISTENCE_DEFECTDOJO_AUTH_KEY | 7fd7eac6fed567b19928f7928a7ddb86f0497e4e |
+| SECURECODEBOX_PERSISTENCE_DEFECTDOJO_AUTH_NAME | admin |
+
+Alternatively the corresponding environment variables, e.g. `SECURECODEBOX_PERSISTENCE_DEFECTDOJO_URL` can be used.
+
# Development
## Local setup
@@ -78,4 +123,5 @@ Well boring yes - but please read our [guidelines and naming standards][scb-deve
[docker]: https://www.docker.com/
[beta-testers]: https://www.securecodebox.io/
+[scb-integration]: https://www.securecodebox.io/integrations
[owasp]: https://www.owasp.org/index.php/Main_Page
diff --git a/pom.xml b/pom.xml
index 28803c4b..bad9dca7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,11 +56,11 @@
IMPORTANT: camunda.version and camunda.spring.boot.starter.version must be compatible
please see org.camunda.bpm.springboot.project:camunda-bpm-spring-boot-starter-root
-->
- 7.10.0
- 3.2.8
+ 7.12.0
+ 3.4.2
- 2.2.2.RELEASE
+ 2.2.6.RELEASE
2.9.2
UTF-8
@@ -101,6 +101,7 @@
org.springframework.boot
spring-boot-properties-migrator
+ ${spring-boot.version}
runtime
@@ -144,7 +145,7 @@
org.camunda.bpm.extension.mockito
camunda-bpm-mockito
test
- 3.2.1
+ 4.12.0
org.camunda.bpm.extension
@@ -155,7 +156,7 @@
org.camunda.bpm.extension
camunda-bpm-assert-scenario
- 0.2
+ 1.0.0
test
@@ -203,7 +204,7 @@
maven-compiler-plugin
- 2.3.1
+ 3.8.1
1.8
1.8
@@ -217,12 +218,12 @@
jcenter-snapshots
jcenter
- http://oss.jfrog.org/artifactory/oss-snapshot-local/
+ https://oss.jfrog.org/artifactory/oss-snapshot-local/
jcenter-releases
jcenter
- http://jcenter.bintray.com
+ https://jcenter.bintray.com
false
@@ -256,7 +257,7 @@
org.owasp
dependency-check-maven
- 5.2.4
+ 5.3.2
ALL
dependency-check-suppression.xml
@@ -315,7 +316,7 @@
org.apache.maven.plugins
maven-source-plugin
- 3.0.1
+ 3.2.1
generate-sources
@@ -328,7 +329,7 @@
org.apache.maven.plugins
maven-javadoc-plugin
- 2.10.4
+ 3.2.0
generate-javadocs
diff --git a/scb-engine/pom.xml b/scb-engine/pom.xml
index 7352b952..d7aacfea 100644
--- a/scb-engine/pom.xml
+++ b/scb-engine/pom.xml
@@ -32,13 +32,12 @@
org.springframework.boot
spring-boot-properties-migrator
runtime
- 2.2.2.RELEASE
org.springframework.security
spring-security-core
- 5.2.2.RELEASE
+ 5.3.1.RELEASE
@@ -90,22 +89,6 @@
tomcat-jdbc
-
- org.apache.tomcat.embed
- tomcat-embed-core
- 9.0.31
-
-
- org.apache.tomcat.embed
- tomcat-embed-el
- 9.0.31
-
-
- org.apache.tomcat.embed
- tomcat-embed-websocket
- 9.0.31
-
-
io.securecodebox.persistenceproviders
empty-persistenceprovider
diff --git a/scb-engine/src/main/resources/application.yaml b/scb-engine/src/main/resources/application.yaml
index 332d6776..af67a640 100644
--- a/scb-engine/src/main/resources/application.yaml
+++ b/scb-engine/src/main/resources/application.yaml
@@ -48,6 +48,15 @@ securecodebox.persistence.elasticsearch.host: persistence-elasticsearch
securecodebox.persistence.elasticsearch.port: 9200
securecodebox.persistence.elasticsearch.index.prefix: securecodebox
securecodebox.persistence.elasticsearch.index.delete_on_init: false
+# Must be 'basic' for basic authentication or 'token' for a api token based authentication
+securecodebox.persistence.elasticsearch.auth: ""
+securecodebox.persistence.elasticsearch.auth.basic.username: ""
+securecodebox.persistence.elasticsearch.auth.basic.password: ""
+securecodebox.persistence.elasticsearch.auth.apikey.id: ""
+securecodebox.persistence.elasticsearch.auth.apikey.secret: ""
+
+# Initialize Kibana with some basic Security Dashboards and Visualisations if no .kibana index will be found on startup
+securecodebox.persistence.elasticsearch.kibana.initialize: true
securecodebox.default.target.name: BodgeIT Public Host
diff --git a/scb-persistenceproviders/defectdojo-persistenceprovider/src/main/java/io/securecodebox/persistence/DefectDojoService.java b/scb-persistenceproviders/defectdojo-persistenceprovider/src/main/java/io/securecodebox/persistence/DefectDojoService.java
index 37e381d3..8069681e 100644
--- a/scb-persistenceproviders/defectdojo-persistenceprovider/src/main/java/io/securecodebox/persistence/DefectDojoService.java
+++ b/scb-persistenceproviders/defectdojo-persistenceprovider/src/main/java/io/securecodebox/persistence/DefectDojoService.java
@@ -26,7 +26,12 @@
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.core.io.ByteArrayResource;
-import org.springframework.http.*;
+
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
import org.springframework.http.converter.FormHttpMessageConverter;
import org.springframework.http.converter.ResourceHttpMessageConverter;
import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
@@ -206,7 +211,7 @@ public EngagementResponse createEngagement(EngagementPayload engagementPayload)
public ImportScanResponse createFindings(String rawResult, long engagementId, long lead, String currentDate, String defectDojoScanName) {
return createFindings(rawResult, engagementId, lead, currentDate,defectDojoScanName, "", new LinkedMultiValueMap<>());
}
- /**
+ /*
* Before version 1.5.4. testName (in DefectDojo _test_type_) must be defectDojoScanName, afterwards, you can have somethings else
*/
public ImportScanResponse createFindings(String rawResult, long engagementId, long lead, String currentDate,String defectDojoScanName, String testName, MultiValueMap options) {
@@ -254,7 +259,7 @@ public String getFilename() {
throw new DefectDojoPersistenceException("Failed to attach findings to engagement.");
}
}
- /**
+ /*
* When DefectDojo >= 1.5.4 is used, testType can be given. Add testName in case DefectDojo >= 1.5.4 is used
* Using testName for each branch leads to multiple issues in DefectDojo, so it is not recommended
*/
@@ -363,7 +368,7 @@ private long getTestIdOrCreate(long engagementId, TestPayload testPayload, Strin
return testId.longValue();
}
- /**
+ /*
* @deprecated
*/
public ImportScanResponse createFindingsReImport(String rawResult, String productName, String engagementName, long lead, String currentDate, String defectDojoScanName, EngagementPayload engagementPayload, TestPayload testPayload, MultiValueMap options) {
@@ -480,7 +485,7 @@ private Optional getEngagementIdByEngagementName(String engagementName, lo
LOG.warn("Engagement with name '{}' not found.", engagementName);
return Optional.empty();
}
- /**
+ /*
* @deprecated
*/
public ProductResponse createProduct(String productName) {
@@ -508,7 +513,9 @@ public void deleteUnusedBranches(List existingBranches, String producNam
/**
* Deletes engagements based on branch tag
- * Be aware that the branch tag MUST be set, otherwise all engagments will be deleted
+ * Be aware that the branch tag MUST be set, otherwise all engagements will be deleted
+ * @param existingBranches The list of existing branches
+ * @param productId The productId to find engagements for
*/
public void deleteUnusedBranches(List existingBranches, long productId) {
if(existingBranches == null) {
diff --git a/scb-persistenceproviders/elasticsearch-persistenceprovider/pom.xml b/scb-persistenceproviders/elasticsearch-persistenceprovider/pom.xml
index 53706a00..77508807 100644
--- a/scb-persistenceproviders/elasticsearch-persistenceprovider/pom.xml
+++ b/scb-persistenceproviders/elasticsearch-persistenceprovider/pom.xml
@@ -31,7 +31,7 @@
0.0.1-SNAPSHOT
- 6.8.7
+ 7.6.2
@@ -53,6 +53,18 @@
${elasticsearch.version}
compile
+
+ org.elasticsearch.client
+ elasticsearch-rest-client
+ ${elasticsearch.version}
+ compile
+
+
+ org.elasticsearch.client
+ transport
+ ${elasticsearch.version}
+ compile
+
org.elasticsearch.test
framework
diff --git a/scb-persistenceproviders/elasticsearch-persistenceprovider/src/main/java/io/securecodebox/persistence/elasticsearch/ElasticSearchPersistenceProvider.java b/scb-persistenceproviders/elasticsearch-persistenceprovider/src/main/java/io/securecodebox/persistence/elasticsearch/ElasticSearchPersistenceProvider.java
index 6a413917..b1a9d766 100644
--- a/scb-persistenceproviders/elasticsearch-persistenceprovider/src/main/java/io/securecodebox/persistence/elasticsearch/ElasticSearchPersistenceProvider.java
+++ b/scb-persistenceproviders/elasticsearch-persistenceprovider/src/main/java/io/securecodebox/persistence/elasticsearch/ElasticSearchPersistenceProvider.java
@@ -27,7 +27,13 @@
import io.securecodebox.model.securitytest.SecurityTest;
import io.securecodebox.persistence.PersistenceException;
import io.securecodebox.persistence.PersistenceProvider;
+import org.apache.http.Header;
import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.message.BasicHeader;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
@@ -37,9 +43,12 @@
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;
+import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
+import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryBuilders;
@@ -55,14 +64,10 @@
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.time.Instant;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
+import java.util.*;
/**
* This component is responsible for persisting the scan-process results in elasticsearch (ES).
@@ -89,6 +94,20 @@ public class ElasticSearchPersistenceProvider implements PersistenceProvider {
@Value("${securecodebox.persistence.elasticsearch.scheme:http}")
private String elasticsearchScheme;
+ @Value("${securecodebox.persistence.elasticsearch.auth}")
+ private String elasticsearchAuth;
+ @Value("${securecodebox.persistence.elasticsearch.auth.basic.username}")
+ private String elasticsearchAuthBasicUsername;
+ @Value("${securecodebox.persistence.elasticsearch.auth.basic.password}")
+ private String elasticsearchAuthBasicPassword;
+ @Value("${securecodebox.persistence.elasticsearch.auth.apikey.id}")
+ private String elasticsearchAuthApiKeyId;
+ @Value("${securecodebox.persistence.elasticsearch.auth.apikey.secret}")
+ private String elasticsearchAuthApiKeySecret;
+
+ // Initialize Kibana with some basic Security Dashboards and Visualisations if no .kibana index will be found on startup
+ @Value("${securecodebox.persistence.elasticsearch.kibana.initialize:true}")
+ private boolean initializeKibana;
/**
* For developing convenience
@@ -111,32 +130,39 @@ public class ElasticSearchPersistenceProvider implements PersistenceProvider {
private void init() {
LOG.info("Initializing ElasticSearchPersistenceProvider");
+
highLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost(elasticsearchHost, elasticsearchPort, elasticsearchScheme)));
+
+ this.handleElasticsearchAuthentication();
+
String indexName = getElasticIndexName();
try {
- connected = highLevelClient.ping();
+ connected = highLevelClient.ping(RequestOptions.DEFAULT);
LOG.debug("ElasticSearch connected?: " + connected);
if (connected) {
if (indexExists(indexName) && deleteOnInit) {
LOG.debug("Deleting Index " + indexName);
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName);
- highLevelClient.indices().delete(deleteIndexRequest);
+ highLevelClient.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT);
}
if (!indexExists(indexName)) {
// The index doesn't exist until now, so we create it
LOG.debug("Index " + indexName + " doesn't exist. Creating it...");
CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
- highLevelClient.indices().create(createIndexRequest);
+ highLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
}
// Checking once more, in case anything went wrong during index creation
if (indexExists(indexName)) {
initialized = true;
- initializeKibana();
+ // Check if Kibana should be initialized with some default default security dashboards
+ if(initializeKibana) {
+ initializeKibana();
+ }
}
} else {
LOG.error("ElasticSearch Host doesn't respond. Please check if it is up and running");
@@ -147,6 +173,53 @@ private void init() {
}
}
+ /**
+ * Handles an authenticated request if authentication parameters are configured.
+ */
+ private void handleElasticsearchAuthentication() {
+
+ if(this.elasticsearchAuth.isEmpty()) {
+ LOG.info("No elasticsearch authentication configured. Trying to connect without authentication");
+ }
+ else {
+ LOG.info("Handling elasticsearch connection authentication for the method: {}", this.elasticsearchAuth);
+
+ if(this.elasticsearchAuth.equals("basic")) {
+ if(!this.elasticsearchAuthBasicUsername.isEmpty() && !this.elasticsearchAuthBasicPassword.isEmpty()) {
+ final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+ credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(this.elasticsearchAuthBasicUsername, this.elasticsearchAuthBasicPassword));
+
+ RestClientBuilder builder = RestClient.builder(
+ new HttpHost(this.elasticsearchHost, this.elasticsearchPort))
+ .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder
+ .setDefaultCredentialsProvider(credentialsProvider));
+ highLevelClient = new RestHighLevelClient(builder);
+ }
+ else {
+ LOG.warn("You need to provide a username and password for the elastic basic auth");
+ }
+ }
+ else if (this.elasticsearchAuth.equals("token")) {
+ if (!this.elasticsearchAuthApiKeyId.isEmpty() && this.elasticsearchAuthApiKeySecret.isEmpty()) {
+ String apiKeyAuth =
+ Base64.getEncoder().encodeToString(
+ (this.elasticsearchAuthApiKeyId + ":" + this.elasticsearchAuthApiKeySecret)
+ .getBytes(StandardCharsets.UTF_8));
+ RestClientBuilder builder = RestClient.builder(
+ new HttpHost("localhost", 9200, "http"));
+ Header[] defaultHeaders =
+ new Header[]{new BasicHeader("Authorization",
+ "ApiKey " + apiKeyAuth)};
+ builder.setDefaultHeaders(defaultHeaders);
+
+ highLevelClient = new RestHighLevelClient(builder);
+ } else {
+ LOG.warn("You need to provide an api token for the elastic token based auth");
+ }
+ }
+ }
+ }
+
@Override
public void persist(SecurityTest securityTest) throws PersistenceException {
@@ -162,7 +235,7 @@ public void persist(SecurityTest securityTest) throws PersistenceException {
}
try {
- connected = highLevelClient.ping();
+ connected = highLevelClient.ping(RequestOptions.DEFAULT);
} catch (IOException ioe) {
LOG.error("Error pinging ElasticSearch: " + ioe.getMessage());
connected = false;
@@ -220,8 +293,8 @@ public void persist(SecurityTest securityTest) throws PersistenceException {
bulkRequest.add(findingIndexRequest);
}
- LOG.info("Persisting SecurityTest and Findings...");
- highLevelClient.bulkAsync(bulkRequest, new ActionListener() {
+ LOG.debug("Persisting SecurityTest and Findings...");
+ highLevelClient.bulkAsync(bulkRequest, RequestOptions.DEFAULT, new ActionListener() {
@Override
public void onResponse(BulkResponse bulkItemResponses) {
if (bulkItemResponses.hasFailures()) {
@@ -234,14 +307,14 @@ public void onResponse(BulkResponse bulkItemResponses) {
@Override
public void onFailure(Exception e) {
- LOG.error("Error persisting findings. Reason: {}", e);
+ LOG.error("Error persisting findings.", e);
throw new ElasticsearchPersistenceException("Request to persist findings to elasticsearch failed.", e);
}
});
} catch (JsonProcessingException e) {
LOG.error(e.getMessage());
} catch (IOException e) {
- throw new ElasticsearchPersistenceException("Error while persisting securityTest into elasticsearch. Is elasticsearch available?.", e);
+ throw new ElasticsearchPersistenceException("Error while persisting securityTest into elasticsearch. Is elasticsearch available?", e);
}
}
@@ -249,14 +322,14 @@ public void onFailure(Exception e) {
* Check if there already is a securityTest persisted under the same uuid.
* This is extremely unlikely but theoretically possible.
*
- * @param securityTest
+ * @param securityTest The securityTest to check the existence for.
*/
private Optional checkForSecurityTestIdExistence(SecurityTest securityTest) throws ElasticsearchPersistenceException, IOException {
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchQuery("id.keyword", securityTest.getId().toString()));
searchRequest.source(searchSourceBuilder);
- SearchResponse searchResponse = highLevelClient.search(searchRequest);
+ SearchResponse searchResponse = highLevelClient.search(searchRequest, RequestOptions.DEFAULT);
LOG.debug("Search Response Status: {}", searchResponse.status());
boolean searchFailure = searchResponse.isTimedOut() || (searchResponse.status() != RestStatus.OK);
@@ -266,7 +339,7 @@ private Optional checkForSecurityTestIdExistence(SecurityTest securityTe
}
LOG.debug("SearchResponse from UUID Search: {}", searchResponse);
- if (searchResponse.getHits().totalHits > 0) {
+ if (searchResponse.getHits().getTotalHits().value > 0) {
return Optional.of(searchResponse.getHits().getAt(0).getId());
}
return Optional.empty();
@@ -291,7 +364,7 @@ private String transformContextForElasticsearchIndexCompatibility() {
/**
* Returns the elasticsearch indexName, based on the current dateTime and configuration.
*
- * @return
+ * @return the elasticsearch indexName
*/
private String getElasticIndexName() {
Date date = Date.from(Instant.now());
@@ -309,10 +382,9 @@ private String getElasticIndexName() {
*/
private boolean indexExists(String indexName) {
+ GetIndexRequest request = new GetIndexRequest(indexName);
try {
- //Indices Exist API is currently not supported in the high level client
- highLevelClient.getLowLevelClient().performRequest("GET", "/" + indexName);
- return true;
+ return this.highLevelClient.indices().exists(request, RequestOptions.DEFAULT);
} catch (ResponseException e) {
if (e.getResponse().getStatusLine().getStatusCode() == 404) {
return false;
@@ -378,7 +450,7 @@ private List