diff --git a/.gitignore b/.gitignore index 908da84c1ecf2d8532bd8cf68f2708e6bf518b0e..78a833095ce074c60d2632bc0355f7c5a5201abd 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ build/ *.tar tmp.yaml .docs/.swagger/api-* +.scannerwork/ # docs .docs/.swagger/dist/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 434cfbb3980d745477508710135965bd0f7370c8..e177cfed10000dd4b7ef1a9d260138eee8563ea1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -272,285 +272,35 @@ test-lib: junit: ./lib/python/report.xml coverage: '/TOTAL.*?([0-9]{1,3})%/' -scan-analyse-service: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master - allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-analyse-service-report.json dbrepo-analyse-service:latest - - trivy image --insecure --exit-code 0 dbrepo-analyse-service:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-analyse-service:latest - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-analyse-service-report.json - -scan-auth-service: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master - allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-authentication-service-report.json dbrepo-auth-service:latest - - trivy image --insecure --exit-code 0 dbrepo-auth-service:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-auth-service:latest - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-authentication-service-report.json - -scan-broker-service: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master - allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-broker-service-report.json dbrepo-broker-service:latest - - trivy image --insecure --exit-code 0 dbrepo-broker-service:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-broker-service:latest - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-broker-service-report.json - -scan-data-db-sidecar: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master - allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-data-db-sidecar-report.json dbrepo-data-db-sidecar:latest - - trivy image --insecure --exit-code 0 data-db-sidecar:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL data-db-sidecar:latest - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-data-db-sidecar-report.json - -scan-gateway-service: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master - allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-gateway-service-report.json docker.io/nginx:1.25.0-alpine-slim - - trivy image --insecure --exit-code 0 docker.io/nginx:1.25.0-alpine-slim - - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/nginx:1.25.0-alpine-slim - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-gateway-service-report.json - -scan-metadata-service: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master - allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-metadata-service-report.json dbrepo-metadata-service:latest - - trivy image --insecure --exit-code 0 dbrepo-metadata-service:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-metadata-service:latest - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-metadata-service-report.json - -scan-data-service: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master - allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-data-service-report.json dbrepo-data-service:latest - - trivy image --insecure --exit-code 0 dbrepo-data-service:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-data-service:latest - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-data-service-report.json - -scan-search-db: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master - allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-search-db-report.json dbrepo-search-db:latest - - trivy image --insecure --exit-code 0 dbrepo-search-db:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-search-db:latest - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-search-db-report.json - -scan-search-service-init: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master - allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-search-service-init-report.json dbrepo-search-service-init:latest - - trivy image --insecure --exit-code 0 dbrepo-search-service-init:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-search-service-init:latest - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-search-service-init-report.json - -scan-data-db: - image: bitnami/trivy:latest +scan-check: stage: scan - only: - refs: - - master - allow_failure: true + image: docker.io/sonarsource/sonar-scanner-cli:10.0 + variables: + SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar" + GIT_DEPTH: "0" script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-data-db-report.json docker.io/bitnami/mariadb:11.2.2-debian-11-r0 - - trivy image --insecure --exit-code 0 docker.io/bitnami/mariadb:11.2.2-debian-11-r0 - - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/bitnami/mariadb:11.2.2-debian-11-r0 - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-data-db-report.json - -scan-ui: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master + - 'sonar-scanner -Dsonar.token="${CI_SONAR_TOKEN}"' allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-ui-report.json dbrepo-ui:latest - - trivy image --insecure --exit-code 0 dbrepo-ui:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-ui:latest cache: + policy: pull + key: "${CI_COMMIT_SHORT_SHA}" paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-ui-report.json + - sonar-scanner/ -scan-search-service: - image: bitnami/trivy:latest +scan-vulnerability: stage: scan - only: - refs: - - master - allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-ui-report.json dbrepo-search-service:latest - - trivy image --insecure --exit-code 0 dbrepo-search-service:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-search-service:latest + image: docker.io/sonarsource/sonar-scanner-cli:10.0 cache: + key: "${CI_COMMIT_SHORT_SHA}" paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-ui-report.json - -scan-storage-service: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master - allow_failure: true + - .sonar/cache script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-storage-service-report.json docker.io/chrislusf/seaweedfs:3.59 - - trivy image --insecure --exit-code 0 docker.io/chrislusf/seaweedfs:3.59 - - trivy image --insecure --exit-code 1 --severity CRITICAL docker.io/chrislusf/seaweedfs:3.59 - cache: - paths: - - .trivycache/ - artifacts: - when: always - expire_in: 1 days - reports: - container_scanning: ./.trivy/trivy-storage-service-report.json - -scan-storage-service-init: - image: bitnami/trivy:latest - stage: scan - only: - refs: - - master + - 'curl -u "${CI_SONAR_TOKEN}:" "${CI_SONAR_URL}/api/issues/gitlab_sast_export?projectKey=DBREPO&branch=${CI_COMMIT_BRANCH}&pullRequest=${CI_MERGE_REQUEST_IID}" -o gl-sast-sonar-report.json' allow_failure: true - script: - - trivy image --insecure --exit-code 0 --format template --template "@.gitlab/gitlab.tpl" -o ./.trivy/trivy-storage-service-init-report.json dbrepo-storage-service-init:latest - - trivy image --insecure --exit-code 0 dbrepo-storage-service-init:latest - - trivy image --insecure --exit-code 1 --severity CRITICAL dbrepo-storage-service-init:latest - cache: - paths: - - .trivycache/ artifacts: - when: always - expire_in: 1 days + expire_in: 1 day reports: - container_scanning: ./.trivy/trivy-storage-service-init-report.json + sast: gl-sast-sonar-report.json docs-registry: stage: docs @@ -652,33 +402,3 @@ release-libs: script: - bash ./lib/python/package.sh - bash ./lib/python/release.sh - -scan-check: - stage: scan - image: docker.io/sonarsource/sonar-scanner-cli:10.0 - variables: - SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar" - GIT_DEPTH: "0" - script: - - 'sonar-scanner -Dsonar.token="${CI_SONAR_TOKEN}" -Dsonar.host.url="${CI_SONAR_URL}"' - allow_failure: true - cache: - policy: pull - key: "${CI_COMMIT_SHORT_SHA}" - paths: - - sonar-scanner/ - -scan-vulnerability: - stage: scan - image: docker.io/sonarsource/sonar-scanner-cli:10.0 - cache: - key: "${CI_COMMIT_SHORT_SHA}" - paths: - - .sonar/cache - script: - - 'curl -u "${CI_SONAR_TOKEN}:" "${CI_SONAR_URL}/api/issues/gitlab_sast_export?projectKey=DBREPO&branch=${CI_COMMIT_BRANCH}&pullRequest=${CI_MERGE_REQUEST_IID}" -o gl-sast-sonar-report.json' - allow_failure: true - artifacts: - expire_in: 1 day - reports: - sast: gl-sast-sonar-report.json \ No newline at end of file diff --git a/README.md b/README.md index 9021e7cf5d6ff10ab494a09da208b515dcd85aaf..e6f58da9748fbf23f034721c7487016e042dc082 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@    + <img src="./dbrepo-ui/public/logo.png" width="200" alt="DBRepo — Repository for Data in Databases" /> diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/MetadataEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/MetadataEndpoint.java index 462ca9df97bd1d9b2aa05e2eac7582ded4f6d64f..886db20df0d917f36e13e7f36c669259a0b5c4d5 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/MetadataEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/MetadataEndpoint.java @@ -5,7 +5,6 @@ import at.tuwien.oaipmh.OaiErrorType; import at.tuwien.oaipmh.OaiListIdentifiersParameters; import at.tuwien.oaipmh.OaiRecordParameters; import at.tuwien.service.MetadataService; -import at.tuwien.utils.XmlUtil; import io.micrometer.observation.annotation.Observed; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -67,7 +66,7 @@ public class MetadataEndpoint { log.debug("endpoint identify repository, verb=Identify"); final String xml = metadataService.identify(); log.trace("identify repository resulted in xml {}", xml); - return ResponseEntity.ok(XmlUtil.pretty(xml)); + return ResponseEntity.ok(xml); } @GetMapping(params = "verb=ListIdentifiers", produces = MediaType.TEXT_XML_VALUE) @@ -77,7 +76,7 @@ public class MetadataEndpoint { log.debug("endpoint list identifiers, verb=ListIdentifiers, parameters={}", parameters); final String xml = metadataService.listIdentifiers(parameters); log.trace("list identifiers resulted in xml {}", xml); - return ResponseEntity.ok(XmlUtil.pretty(xml)); + return ResponseEntity.ok(xml); } @GetMapping(params = "verb=GetRecord", produces = MediaType.TEXT_XML_VALUE) @@ -90,28 +89,28 @@ public class MetadataEndpoint { log.trace("metadataPrefix does not match supported list: {}", supportedMetadataFormats); log.error("Failed to get record: Format {} is not supported", parameters.getMetadataPrefix()); return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(XmlUtil.pretty(metadataService.error(OaiErrorType.CANNOT_DISSEMINATE_FORMAT))); + .body(metadataService.error(OaiErrorType.CANNOT_DISSEMINATE_FORMAT)); } log.trace("metadata prefix {} is supported", parameters.getMetadataPrefix()); final List<String> supportedIdentifierPrefixes = List.of("doi", "oai"); if (parameters.getIdentifier() == null) { log.error("Failed to get record: Identifier is empty"); return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(XmlUtil.pretty(metadataService.error(OaiErrorType.NO_RECORDS_MATCH))); + .body(metadataService.error(OaiErrorType.NO_RECORDS_MATCH)); } else if (supportedIdentifierPrefixes.stream().noneMatch(identifierPrefix -> parameters.getIdentifier().startsWith(identifierPrefix)) || parameters.getIdentifier().indexOf(':') > 3) { log.error("Failed to get record: Identifier does not match supported prefixes {}", supportedIdentifierPrefixes); return ResponseEntity.status(HttpStatus.BAD_REQUEST) - .body(XmlUtil.pretty(metadataService.error(OaiErrorType.NO_RECORDS_MATCH))); + .body(metadataService.error(OaiErrorType.NO_RECORDS_MATCH)); } log.trace("identifier prefix of {} is supported", parameters.getIdentifier()); try { final String xml = metadataService.getRecord(parameters); log.trace("get record resulted in xml {}", xml); - return ResponseEntity.ok(XmlUtil.pretty(xml)); + return ResponseEntity.ok(xml); } catch (IdentifierNotFoundException e) { return ResponseEntity.status(HttpStatus.NOT_FOUND) - .body(XmlUtil.pretty(metadataService.error(OaiErrorType.ID_DOES_NOT_EXIST))); + .body(metadataService.error(OaiErrorType.ID_DOES_NOT_EXIST)); } } @@ -122,7 +121,7 @@ public class MetadataEndpoint { log.debug("endpoint list metadata formats, verb=ListMetadataFormats"); final String xml = metadataService.listMetadataFormats(); log.trace("list metadata formats resulted in xml {}", xml); - return ResponseEntity.ok(XmlUtil.pretty(xml)); + return ResponseEntity.ok(xml); } } diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MetadataServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MetadataServiceImpl.java index 9dfefd18cca20a2f6c2e90182c112af147930f18..14c621e2f9d10799864d8d85a6f019353ca67b08 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MetadataServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MetadataServiceImpl.java @@ -17,7 +17,6 @@ import at.tuwien.oaipmh.OaiRecordParameters; import at.tuwien.repository.IdentifierRepository; import at.tuwien.service.IdentifierService; import at.tuwien.service.MetadataService; -import at.tuwien.utils.XmlUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -120,7 +119,7 @@ public class MetadataServiceImpl implements MetadataService { final StringBuilder builder = new StringBuilder("<ListMetadataFormats>"); builder.append(templateEngine.process("metadata-format.xml", new Context())); builder.append("</ListMetadataFormats>"); - return XmlUtil.pretty(parseResponse("verb=\"ListMetadataFormats\"", builder.toString())); + return parseResponse("verb=\"ListMetadataFormats\"", builder.toString()); } @Override @@ -130,7 +129,7 @@ public class MetadataServiceImpl implements MetadataService { context.setVariable("message", type.getErrorText()); final String body = templateEngine.process("error.xml", context); log.trace("mapped error {}", type); - return XmlUtil.pretty(parseResponse(body)); + return parseResponse(body); } private String requestUrl() { @@ -153,7 +152,7 @@ public class MetadataServiceImpl implements MetadataService { context.setVariable("request", "<request " + parameterString + ">" + requestUrl() + "</request>"); } context.setVariable("body", body); - return XmlUtil.pretty(templateEngine.process("_header.xml", context)); + return templateEngine.process("_header.xml", context); } @Override diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/utils/XmlUtil.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/utils/XmlUtil.java deleted file mode 100644 index 42db2b93797e9a3838d54fb790c05e853e5d738d..0000000000000000000000000000000000000000 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/utils/XmlUtil.java +++ /dev/null @@ -1,42 +0,0 @@ -package at.tuwien.utils; - -import org.w3c.dom.Document; -import org.xml.sax.InputSource; - -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.StringReader; -import java.io.StringWriter; -import java.io.Writer; - -public class XmlUtil { - - public static String pretty(String xmlString) { - return pretty(xmlString, 2, true); - } - - public static String pretty(String xmlString, int indent, boolean ignoreDeclaration) { - xmlString = xmlString.replaceAll("(?m)^[ \t]*\r?\n", "").replaceAll("> <", "><"); - try { - final InputSource src = new InputSource(new StringReader(xmlString.trim())); - final Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(src); - final TransformerFactory transformerFactory = TransformerFactory.newInstance(); - transformerFactory.setAttribute("indent-number", indent); - final Transformer transformer = transformerFactory.newTransformer(); - transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); - transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, ignoreDeclaration ? "yes" : "no"); - transformer.setOutputProperty(OutputKeys.INDENT, "np"); - final Writer out = new StringWriter(); - transformer.transform(new DOMSource(document), new StreamResult(out)); - return out.toString() - .trim(); - } catch (Exception e) { - throw new RuntimeException("Error occurs when pretty-printing xml:\n" + xmlString, e); - } - } - -} diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000000000000000000000000000000000000..77c810f71bc0beba8192e97eeec8283752d6f3aa --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,4 @@ +sonar.projectKey=DBREPO +sonar.host.url=http://57.153.70.97:9000 +sonar.sources=./dbrepo-metadata-service/test/src/main,./dbrepo-metadata-service/services/src/main,./dbrepo-metadata-service/repositories/src/main,./dbrepo-metadata-service/rest-service/src/main,./dbrepo-metadata-service/api/src/main,./dbrepo-metadata-service/oai/src/main,./dbrepo-metadata-service/entities/src/main,./dbrepo-data-service/services/src/main,./dbrepo-data-service/rest-service/src/main,./dbrepo-data-service/querystore/src/main +sonar.java.binaries=./dbrepo-metadata-service/test/target/classes,./dbrepo-metadata-service/services/target/classes,./dbrepo-metadata-service/repositories/target/classes,./dbrepo-metadata-service/rest-service/target/classes,./dbrepo-metadata-service/api/target/classes,./dbrepo-metadata-service/oai/target/classes,./dbrepo-metadata-service/entities/target/classes,./dbrepo-data-service/services/target/classes,./dbrepo-data-service/rest-service/target/classes,./dbrepo-data-service/querystore/target/classes \ No newline at end of file