Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • fair-data-austria-db-repository/fda-services
1 result
Select Git revision
Show changes
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 892 additions and 492 deletions
dbrepo-dashboard-service/report.xml deleted 100644 → 0
View file @ 79a3df8a
<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="48" time="25.604" timestamp="2025-01-29T15:46:14.797988+01:00" hostname="medusa"><testcase classname="test.test_app.JwtTest" name="test_delete_database_no_auth_fails" time="11.131" /><testcase classname="test.test_app.JwtTest" name="test_delete_database_no_role_fails" time="0.317" /><testcase classname="test.test_app.JwtTest" name="test_delete_database_not_found_fails" time="0.370" /><testcase classname="test.test_app.JwtTest" name="test_delete_database_succeeds" time="0.524" /><testcase classname="test.test_app.JwtTest" name="test_get_fields_fails" time="0.154" /><testcase classname="test.test_app.JwtTest" name="test_get_fields_succeeds" time="0.218" /><testcase classname="test.test_app.JwtTest" name="test_get_fuzzy_search_no_query_fails" time="0.173" /><testcase classname="test.test_app.JwtTest" name="test_get_fuzzy_search_succeeds" time="0.283" /><testcase classname="test.test_app.JwtTest" name="test_get_index_fails" time="0.240" /><testcase classname="test.test_app.JwtTest" name="test_get_index_succeeds" time="0.190" /><testcase classname="test.test_app.JwtTest" name="test_health_succeeds" time="0.160" /><testcase classname="test.test_app.JwtTest" name="test_post_general_search_column_succeeds" time="0.386" /><testcase classname="test.test_app.JwtTest" name="test_post_general_search_concept_succeeds" time="0.341" /><testcase classname="test.test_app.JwtTest" name="test_post_general_search_identifier_succeeds" time="0.312" /><testcase classname="test.test_app.JwtTest" name="test_post_general_search_media_type_fails" time="0.140" /><testcase classname="test.test_app.JwtTest" name="test_post_general_search_no_body_fails" time="0.134" /><testcase classname="test.test_app.JwtTest" name="test_post_general_search_succeeds" time="0.284" /><testcase classname="test.test_app.JwtTest" name="test_post_general_search_table_succeeds" time="0.336" /><testcase classname="test.test_app.JwtTest" name="test_post_general_search_unit_succeeds" time="0.246" /><testcase classname="test.test_app.JwtTest" name="test_post_general_search_view_succeeds" time="0.281" /><testcase classname="test.test_app.JwtTest" name="test_update_database_empty_body_fails" time="0.177" /><testcase classname="test.test_app.JwtTest" name="test_update_database_malformed_body_fails" time="0.180" /><testcase classname="test.test_app.JwtTest" name="test_update_database_media_type_fails" time="0.231" /><testcase classname="test.test_app.JwtTest" name="test_update_database_no_auth_fails" time="0.119" /><testcase classname="test.test_app.JwtTest" name="test_update_database_no_body_fails" time="0.150" /><testcase classname="test.test_app.JwtTest" name="test_update_database_succeeds" time="0.243" /><testcase classname="test.test_jwt.JwtTest" name="test_get_user_roles_succeeds" time="0.146" /><testcase classname="test.test_jwt.JwtTest" name="test_verify_password_empty_password_fails" time="0.144" /><testcase classname="test.test_jwt.JwtTest" name="test_verify_password_empty_username_fails" time="0.127" /><testcase classname="test.test_jwt.JwtTest" name="test_verify_password_no_password_fails" time="0.142" /><testcase classname="test.test_jwt.JwtTest" name="test_verify_password_no_username_fails" time="0.146" /><testcase classname="test.test_jwt.JwtTest" name="test_verify_password_succeeds" time="0.152" /><testcase classname="test.test_jwt.JwtTest" name="test_verify_token_empty_token_fails" time="0.144" /><testcase classname="test.test_jwt.JwtTest" name="test_verify_token_malformed_token_fails" time="0.143" /><testcase classname="test.test_jwt.JwtTest" name="test_verify_token_no_token_fails" time="0.130" /><testcase classname="test.test_jwt.JwtTest" name="test_verify_token_succeeds" time="0.212" /><testcase classname="test.test_opensearch_client.OpenSearchClientTest" name="test_delete_database_fails" time="0.120" /><testcase classname="test.test_opensearch_client.OpenSearchClientTest" name="test_delete_database_succeeds" time="0.172" /><testcase classname="test.test_opensearch_client.OpenSearchClientTest" name="test_fuzzy_search_succeeds" time="0.190" /><testcase classname="test.test_opensearch_client.OpenSearchClientTest" name="test_get_fields_for_index_database_succeeds" time="0.201" /><testcase classname="test.test_opensearch_client.OpenSearchClientTest" name="test_get_fields_for_index_user_succeeds" time="0.202" /><testcase classname="test.test_opensearch_client.OpenSearchClientTest" name="test_unit_independent_search_fails" time="0.208" /><testcase classname="test.test_opensearch_client.OpenSearchClientTest" name="test_update_database_create_succeeds" time="0.205" /><testcase classname="test.test_opensearch_client.OpenSearchClientTest" name="test_update_database_malformed_fails" time="0.237" /><testcase classname="test.test_opensearch_client.OpenSearchClientTest" name="test_update_database_succeeds" time="0.214" /><testcase classname="test.test_keycloak_client.JwtTest" name="test_obtain_user_token_malformed_fails" time="0.112" /><testcase classname="test.test_keycloak_client.JwtTest" name="test_obtain_user_token_succeeds" time="0.149" /><testcase classname="test.test_keycloak_client.JwtTest" name="test_verify_jwt_succeeds" time="0.684" /></testsuite></testsuites>
\ No newline at end of file
dbrepo-data-service/pom.xml
View file @ 841d4894
......@@ -8,10 +8,15 @@
<version>3.3.5</version>
</parent>
<organization>
<name>TU Wien</name>
<url>https://www.tuwien.ac.at</url>
</organization>
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-data-service</artifactId>
<name>dbrepo-data-service</name>
<version>1.6.5</version>
<version>1.7.0</version>
<description>Service that manages the data</description>
......@@ -23,7 +28,7 @@
<module>report</module>
</modules>
<url>https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.6/</url>
<url>https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.7/</url>
<developers>
<developer>
<name>Martin Weise</name>
......@@ -54,14 +59,37 @@
<minio.version>8.5.7</minio.version>
<guava.version>33.3.0-jre</guava.version>
<spark.version>4.0.0-preview2</spark.version>
<keycloak.version>26.0.4</keycloak.version>
<scala.version>2.13</scala.version>
<antlr-runtime.version>3.5.2</antlr-runtime.version>
<micrometer.version>1.10.0</micrometer.version>
<!-- see https://github.com/apache/spark/blob/cde8e4a82e20a363861f451ebd5138efb3194ab8/pom.xml -->
<hadoop.version>3.4.0</hadoop.version>
<jakarta-servlet.version>5.0.0</jakarta-servlet.version>
<sonar.coverage.jacoco.xmlReportPaths>./report/target/site/jacoco-aggregate/jacoco.xml
<sonar.coverage.jacoco.xmlReportPaths>
./report/target/site/jacoco-aggregate/jacoco.xml
</sonar.coverage.jacoco.xmlReportPaths>
<CodeCacheSize>128m</CodeCacheSize>
<extraJavaTestArgs>
-XX:+IgnoreUnrecognizedVMOptions
--add-modules=jdk.incubator.vector
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
--add-opens=java.base/sun.security.action=ALL-UNNAMED
--add-opens=java.base/sun.util.calendar=ALL-UNNAMED
-Djdk.reflect.useDirectMethodHandle=false
-Dio.netty.tryReflectionSetAccessible=true
</extraJavaTestArgs>
</properties>
<dependencies>
......@@ -208,6 +236,22 @@
<artifactId>commons-validator</artifactId>
<version>${commons-validator.version}</version>
</dependency>
<dependency>
<groupId>org.jooq</groupId>
<artifactId>jooq</artifactId>
<version>${jooq.version}</version>
</dependency>
<!-- Authentication -->
<dependency>
<groupId>org.keycloak</groupId>
<artifactId>keycloak-common</artifactId>
<version>${keycloak.version}</version>
</dependency>
<dependency>
<groupId>org.keycloak</groupId>
<artifactId>keycloak-admin-client</artifactId>
<version>${keycloak.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-hibernate6</artifactId>
......@@ -323,7 +367,26 @@
</execution>
</executions>
</plugin>
<!-- Surefire runs all Java tests -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<!-- Note config is repeated in scalatest config -->
<configuration>
<argLine>@{argLine} -ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize}
${extraJavaTestArgs}
</argLine>
</configuration>
</plugin>
</plugins>
</build>
<licenses>
<license>
<name>Apache-2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
<distribution>repo</distribution>
</license>
</licenses>
</project>
dbrepo-data-service/querystore/pom.xml
View file @ 841d4894
......@@ -6,12 +6,12 @@
<parent>
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-data-service</artifactId>
<version>1.6.5</version>
<version>1.7.0</version>
</parent>
<artifactId>dbrepo-data-service-querystore</artifactId>
<name>dbrepo-data-service-querystore</name>
<version>1.6.5</version>
<version>1.7.0</version>
<dependencies/>
......
dbrepo-data-service/report/pom.xml
View file @ 841d4894
......@@ -6,12 +6,12 @@
<parent>
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-data-service</artifactId>
<version>1.6.5</version>
<version>1.7.0</version>
</parent>
<artifactId>report</artifactId>
<name>dbrepo-data-service-report</name>
<version>1.6.5</version>
<version>1.7.0</version>
<description>
This module is only intended for the pipeline coverage report. See the detailed report in the
respective modules
......
dbrepo-data-service/rest-service/pom.xml
View file @ 841d4894
......@@ -6,18 +6,18 @@
<parent>
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-data-service</artifactId>
<version>1.6.5</version>
<version>1.7.0</version>
</parent>
<artifactId>rest-service</artifactId>
<name>dbrepo-data-service-rest-service</name>
<version>1.6.5</version>
<version>1.7.0</version>
<dependencies>
<dependency>
<groupId>at.tuwien</groupId>
<artifactId>services</artifactId>
<version>1.6.5</version>
<version>1.7.0</version>
</dependency>
</dependencies>
......
dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java
View file @ 841d4894
......@@ -6,7 +6,7 @@ import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.api.user.UserDto;
import at.tuwien.exception.*;
import at.tuwien.service.AccessService;
import at.tuwien.service.CredentialService;
import at.tuwien.service.CacheService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
......@@ -31,13 +31,13 @@ import java.util.UUID;
@RequestMapping(path = "/api/database/{databaseId}/access")
public class AccessEndpoint extends RestEndpoint {
private final CacheService cacheService;
private final AccessService accessService;
private final CredentialService credentialService;
@Autowired
public AccessEndpoint(AccessService accessService, CredentialService credentialService) {
public AccessEndpoint(CacheService cacheService, AccessService accessService) {
this.cacheService = cacheService;
this.accessService = accessService;
this.credentialService = credentialService;
}
@PostMapping("/{userId}")
......@@ -74,14 +74,14 @@ public class AccessEndpoint extends RestEndpoint {
mediaType = "application/json",
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<Void> create(@NotNull @PathVariable("databaseId") Long databaseId,
public ResponseEntity<Void> create(@NotNull @PathVariable("databaseId") UUID databaseId,
@PathVariable("userId") UUID userId,
@Valid @RequestBody CreateAccessDto data)
throws NotAllowedException, DatabaseUnavailableException, DatabaseNotFoundException,
RemoteUnavailableException, UserNotFoundException, DatabaseMalformedException, MetadataServiceException {
log.debug("endpoint give access to database, databaseId={}, userId={}", databaseId, userId);
final DatabaseDto database = credentialService.getDatabase(databaseId);
final UserDto user = credentialService.getUser(userId);
final DatabaseDto database = cacheService.getDatabase(databaseId);
final UserDto user = cacheService.getUser(userId);
if (database.getAccesses().stream().anyMatch(a -> a.getUser().getId().equals(userId))) {
log.error("Failed to create access to user with id {}: already has access", userId);
throw new NotAllowedException("Failed to create access to user with id " + userId + ": already has access");
......@@ -130,15 +130,15 @@ public class AccessEndpoint extends RestEndpoint {
mediaType = "application/json",
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<Void> update(@NotNull @PathVariable("databaseId") Long databaseId,
public ResponseEntity<Void> update(@NotNull @PathVariable("databaseId") UUID databaseId,
@PathVariable("userId") UUID userId,
@Valid @RequestBody CreateAccessDto access) throws NotAllowedException,
DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException, UserNotFoundException,
DatabaseMalformedException, MetadataServiceException {
log.debug("endpoint modify access to database, databaseId={}, userId={}, access.type={}", databaseId, userId,
access.getType());
final DatabaseDto database = credentialService.getDatabase(databaseId);
final UserDto user = credentialService.getUser(userId);
final DatabaseDto database = cacheService.getDatabase(databaseId);
final UserDto user = cacheService.getUser(userId);
if (database.getAccesses().stream().noneMatch(a -> a.getHuserid().equals(userId))) {
log.error("Failed to update access to user with id {}: no access", userId);
throw new NotAllowedException("Failed to update access to user with id " + userId + ": no access");
......@@ -153,22 +153,6 @@ public class AccessEndpoint extends RestEndpoint {
}
}
@PutMapping
@PreAuthorize("hasAuthority('system')")
@Operation(summary = "Invalidate access cache for database",
security = {@SecurityRequirement(name = "basicAuth")},
hidden = true)
@ApiResponses(value = {
@ApiResponse(responseCode = "202",
description = "Invalidated access cache succeeded")
})
public ResponseEntity<Void> invalidateAccess(@NotNull @PathVariable("databaseId") Long databaseId) {
log.debug("endpoint empty access cache for database, databaseId={}", databaseId);
credentialService.invalidateAccess(databaseId);
return ResponseEntity.accepted()
.build();
}
@DeleteMapping("/{userId}")
@PreAuthorize("hasAuthority('system')")
@Operation(summary = "Revoke access",
......@@ -203,13 +187,13 @@ public class AccessEndpoint extends RestEndpoint {
mediaType = "application/json",
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<Void> revoke(@NotNull @PathVariable("databaseId") Long databaseId,
public ResponseEntity<Void> revoke(@NotNull @PathVariable("databaseId") UUID databaseId,
@PathVariable("userId") UUID userId) throws NotAllowedException,
DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException, UserNotFoundException,
DatabaseMalformedException, MetadataServiceException {
log.debug("endpoint revoke access to database, databaseId={}, userId={}", databaseId, userId);
final DatabaseDto database = credentialService.getDatabase(databaseId);
final UserDto user = credentialService.getUser(userId);
final DatabaseDto database = cacheService.getDatabase(databaseId);
final UserDto user = cacheService.getUser(userId);
if (database.getAccesses().stream().noneMatch(a -> a.getUser().getId().equals(userId))) {
log.error("Failed to delete access to user with id {}: no access", userId);
throw new NotAllowedException("Failed to delete access to user with id " + userId + ": no access");
......
dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/DatabaseEndpoint.java
View file @ 841d4894
......@@ -10,7 +10,7 @@ import at.tuwien.api.user.internal.UpdateUserPasswordDto;
import at.tuwien.exception.*;
import at.tuwien.service.AccessService;
import at.tuwien.service.ContainerService;
import at.tuwien.service.CredentialService;
import at.tuwien.service.CacheService;
import at.tuwien.service.DatabaseService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
......@@ -28,6 +28,7 @@ import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import java.sql.SQLException;
import java.util.UUID;
@Log4j2
@RestController
......@@ -35,18 +36,18 @@ import java.sql.SQLException;
@RequestMapping(path = "/api/database")
public class DatabaseEndpoint extends RestEndpoint {
private final CacheService cacheService;
private final AccessService accessService;
private final DatabaseService databaseService;
private final ContainerService containerService;
private final CredentialService credentialService;
@Autowired
public DatabaseEndpoint(AccessService accessService, DatabaseService databaseService,
ContainerService containerService, CredentialService credentialService) {
public DatabaseEndpoint(CacheService cacheService, AccessService accessService, DatabaseService databaseService,
ContainerService containerService) {
this.cacheService = cacheService;
this.accessService = accessService;
this.databaseService = databaseService;
this.containerService = containerService;
this.credentialService = credentialService;
}
@PostMapping
......@@ -86,7 +87,7 @@ public class DatabaseEndpoint extends RestEndpoint {
DatabaseMalformedException, QueryStoreCreateException, MetadataServiceException {
log.debug("endpoint create database, data.containerId={}, data.internalName={}, data.username={}",
data.getContainerId(), data.getInternalName(), data.getUsername());
final ContainerDto container = credentialService.getContainer(data.getContainerId());
final ContainerDto container = cacheService.getContainer(data.getContainerId());
try {
final DatabaseDto database = containerService.createDatabase(container, data);
containerService.createQueryStore(container, data.getInternalName());
......@@ -128,13 +129,13 @@ public class DatabaseEndpoint extends RestEndpoint {
mediaType = "application/json",
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<Void> update(@NotNull @PathVariable("databaseId") Long databaseId,
public ResponseEntity<Void> update(@NotNull @PathVariable("databaseId") UUID databaseId,
@Valid @RequestBody UpdateUserPasswordDto data)
throws DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException,
DatabaseMalformedException, MetadataServiceException {
log.debug("endpoint update user password in database, databaseId={}, data.username={}", databaseId,
data.getUsername());
final DatabaseDto database = credentialService.getDatabase(databaseId);
final DatabaseDto database = cacheService.getDatabase(databaseId);
try {
databaseService.update(database, data);
return ResponseEntity.status(HttpStatus.ACCEPTED)
......
dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
View file @ 841d4894
package at.tuwien.endpoints;
import at.tuwien.ExportResourceDto;
import at.tuwien.api.database.CreateViewDto;
import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.ViewColumnDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.query.ExecuteStatementDto;
import at.tuwien.api.database.query.QueryDto;
import at.tuwien.api.database.query.QueryPersistDto;
import at.tuwien.api.database.query.SubsetDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.mapper.MetadataMapper;
import at.tuwien.service.*;
import at.tuwien.service.CacheService;
import at.tuwien.service.DatabaseService;
import at.tuwien.service.StorageService;
import at.tuwien.service.SubsetService;
import at.tuwien.validation.EndpointValidator;
import io.micrometer.observation.annotation.Observed;
import io.swagger.v3.oas.annotations.Operation;
......@@ -50,27 +53,27 @@ import java.util.UUID;
@RequestMapping(path = "/api/database/{databaseId}/subset")
public class SubsetEndpoint extends RestEndpoint {
private final CacheService cacheService;
private final MariaDbMapper mariaDbMapper;
private final SubsetService subsetService;
private final MetadataMapper metadataMapper;
private final MetricsService metricsService;
private final StorageService storageService;
private final DatabaseService databaseService;
private final CredentialService credentialService;
private final EndpointValidator endpointValidator;
private final MetadataServiceGateway metadataServiceGateway;
@Autowired
public SubsetEndpoint(MariaDbMapper mariaDbMapper, SubsetService subsetService, MetadataMapper metadataMapper,
MetricsService metricsService, StorageService storageService, DatabaseService databaseService,
CredentialService credentialService, EndpointValidator endpointValidator) {
public SubsetEndpoint(CacheService cacheService, MariaDbMapper mariaDbMapper, SubsetService subsetService,
MetadataMapper metadataMapper, StorageService storageService, DatabaseService databaseService,
EndpointValidator endpointValidator, MetadataServiceGateway metadataServiceGateway) {
this.cacheService = cacheService;
this.mariaDbMapper = mariaDbMapper;
this.subsetService = subsetService;
this.metadataMapper = metadataMapper;
this.metricsService = metricsService;
this.storageService = storageService;
this.databaseService = databaseService;
this.credentialService = credentialService;
this.endpointValidator = endpointValidator;
this.metadataServiceGateway = metadataServiceGateway;
}
@GetMapping
......@@ -100,13 +103,13 @@ public class SubsetEndpoint extends RestEndpoint {
mediaType = "application/json",
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<List<QueryDto>> list(@NotNull @PathVariable("databaseId") Long databaseId,
public ResponseEntity<List<QueryDto>> list(@NotNull @PathVariable("databaseId") UUID databaseId,
@RequestParam(name = "persisted", required = false) Boolean filterPersisted,
Principal principal)
throws DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException,
QueryNotFoundException, NotAllowedException, MetadataServiceException {
log.debug("endpoint find subsets in database, databaseId={}, filterPersisted={}", databaseId, filterPersisted);
final DatabaseDto database = credentialService.getDatabase(databaseId);
final DatabaseDto database = cacheService.getDatabase(databaseId);
endpointValidator.validateOnlyPrivateSchemaAccess(database, principal);
final List<QueryDto> queries;
try {
......@@ -157,8 +160,8 @@ public class SubsetEndpoint extends RestEndpoint {
mediaType = "application/json",
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<?> findById(@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("subsetId") Long subsetId,
public ResponseEntity<?> findById(@NotNull @PathVariable("databaseId") UUID databaseId,
@NotNull @PathVariable("subsetId") UUID subsetId,
@NotNull @RequestHeader("Accept") String accept,
@RequestParam(required = false) Instant timestamp,
Principal principal)
......@@ -167,7 +170,7 @@ public class SubsetEndpoint extends RestEndpoint {
MetadataServiceException, TableNotFoundException, QueryMalformedException, NotAllowedException {
log.debug("endpoint find subset in database, databaseId={}, subsetId={}, accept={}, timestamp={}", databaseId,
subsetId, accept, timestamp);
final DatabaseDto database = credentialService.getDatabase(databaseId);
final DatabaseDto database = cacheService.getDatabase(databaseId);
endpointValidator.validateOnlyPrivateSchemaAccess(database, principal);
final QueryDto subset;
try {
......@@ -181,7 +184,7 @@ public class SubsetEndpoint extends RestEndpoint {
timestamp = Instant.now();
log.debug("timestamp not set: default to {}", timestamp);
}
if (accept == null || accept.isEmpty() || accept.isBlank()) {
if (accept == null || accept.isBlank()) {
accept = MediaType.APPLICATION_JSON_VALUE;
log.debug("accept header not set: default to {}", accept);
}
......@@ -192,8 +195,7 @@ public class SubsetEndpoint extends RestEndpoint {
case "text/csv":
log.trace("accept header matches csv");
final String query = mariaDbMapper.rawSelectQuery(subset.getQuery(), timestamp, null, null);
final Dataset<Row> dataset = subsetService.getData(database, query, timestamp, null, null, null, null);
metricsService.countSubsetGetData(databaseId, subsetId);
final Dataset<Row> dataset = subsetService.getData(database, query);
final ExportResourceDto resource = storageService.transformDataset(dataset);
final HttpHeaders headers = new HttpHeaders();
headers.add("Content-Disposition", "attachment; filename=\"" + resource.getFilename() + "\"");
......@@ -247,8 +249,8 @@ public class SubsetEndpoint extends RestEndpoint {
mediaType = "application/json",
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<List<Map<String, Object>>> create(@NotNull @PathVariable("databaseId") Long databaseId,
@Valid @RequestBody ExecuteStatementDto data,
public ResponseEntity<List<Map<String, Object>>> create(@NotNull @PathVariable("databaseId") UUID databaseId,
@Valid @RequestBody SubsetDto data,
Principal principal,
@NotNull HttpServletRequest request,
@RequestParam(required = false) Instant timestamp,
......@@ -258,13 +260,12 @@ public class SubsetEndpoint extends RestEndpoint {
QueryNotFoundException, StorageUnavailableException, QueryMalformedException, StorageNotFoundException,
QueryStoreInsertException, TableMalformedException, PaginationException, QueryNotSupportedException,
NotAllowedException, UserNotFoundException, MetadataServiceException, TableNotFoundException,
ViewMalformedException, ViewNotFoundException {
log.debug("endpoint create subset in database, databaseId={}, data.statement={}, page={}, size={}, " +
"timestamp={}", databaseId, data.getStatement(), page, size,
timestamp);
ViewMalformedException, ViewNotFoundException, ImageNotFoundException {
log.debug("endpoint create subset in database, databaseId={}, page={}, size={}, timestamp={}", databaseId,
page, size, timestamp);
/* check */
endpointValidator.validateDataParams(page, size);
endpointValidator.validateForbiddenStatements(data.getStatement());
endpointValidator.validateSubsetParams(data);
/* parameters */
final UUID userId;
if (principal != null) {
......@@ -285,10 +286,10 @@ public class SubsetEndpoint extends RestEndpoint {
log.debug("timestamp not set: default to {}", timestamp);
}
/* create */
final DatabaseDto database = credentialService.getDatabase(databaseId);
final DatabaseDto database = cacheService.getDatabase(databaseId);
endpointValidator.validateOnlyPrivateSchemaAccess(database, principal);
try {
final Long subsetId = subsetService.create(database, data.getStatement(), timestamp, userId);
final UUID subsetId = subsetService.create(database, data, timestamp, userId);
return getData(databaseId, subsetId, principal, request, timestamp, page, size);
} catch (SQLException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
......@@ -304,9 +305,9 @@ public class SubsetEndpoint extends RestEndpoint {
@ApiResponses(value = {
@ApiResponse(responseCode = "200",
description = "Retrieved subset data",
headers = {@Header(name = "X-Count", description = "Number of rows", schema = @Schema(implementation = Long.class)),
headers = {@Header(name = "X-Count", description = "Number of rows", schema = @Schema(implementation = UUID.class)),
@Header(name = "X-Headers", description = "The list of headers separated by comma", schema = @Schema(implementation = String.class)),
@Header(name = "X-Id", description = "The subset id", schema = @Schema(implementation = Long.class), required = true),
@Header(name = "X-Id", description = "The subset id", schema = @Schema(implementation = UUID.class), required = true),
@Header(name = "Access-Control-Expose-Headers", description = "Reverse proxy exposing of custom headers", schema = @Schema(implementation = String.class), required = true)},
content = {@Content(
mediaType = "application/json",
......@@ -332,26 +333,26 @@ public class SubsetEndpoint extends RestEndpoint {
mediaType = "application/json",
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<List<Map<String, Object>>> getData(@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("subsetId") Long subsetId,
public ResponseEntity<List<Map<String, Object>>> getData(@NotNull @PathVariable("databaseId") UUID databaseId,
@NotNull @PathVariable("subsetId") UUID subsetId,
Principal principal,
@NotNull HttpServletRequest request,
@RequestParam(required = false) Instant timestamp,
@RequestParam(required = false) Long page,
@RequestParam(required = false) Long size)
throws PaginationException, DatabaseNotFoundException, RemoteUnavailableException, NotAllowedException,
QueryNotFoundException, DatabaseUnavailableException, TableMalformedException, QueryMalformedException,
UserNotFoundException, MetadataServiceException, TableNotFoundException, ViewNotFoundException, ViewMalformedException {
QueryNotFoundException, DatabaseUnavailableException, QueryMalformedException, UserNotFoundException,
MetadataServiceException, TableNotFoundException, ViewNotFoundException, ViewMalformedException {
log.debug("endpoint get subset data, databaseId={}, subsetId={}, principal.name={} page={}, size={}",
databaseId, subsetId, principal != null ? principal.getName() : null, page, size);
endpointValidator.validateDataParams(page, size);
final DatabaseDto database = credentialService.getDatabase(databaseId);
final DatabaseDto database = cacheService.getDatabase(databaseId);
if (!database.getIsPublic()) {
if (principal == null) {
log.error("Failed to re-execute query: no authentication found");
throw new NotAllowedException("Failed to re-execute query: no authentication found");
}
credentialService.getAccess(databaseId, getId(principal));
cacheService.getAccess(databaseId, getId(principal));
}
log.trace("visibility for database: is_public={}, is_schema_public={}", database.getIsPublic(), database.getIsSchemaPublic());
/* parameters */
......@@ -379,16 +380,11 @@ public class SubsetEndpoint extends RestEndpoint {
.headers(headers)
.build();
}
subset.setIdentifiers(metadataServiceGateway.getIdentifiers(database.getId(), subset.getId()));
final String query = mariaDbMapper.rawSelectQuery(subset.getQuery(), timestamp, page, size);
final Dataset<Row> dataset = subsetService.getData(database, query, timestamp, page, size, null, null);
metricsService.countSubsetGetData(databaseId, subsetId);
final Dataset<Row> dataset = subsetService.getData(database, query);
final String viewName = metadataMapper.queryDtoToViewName(subset);
databaseService.createView(database, CreateViewDto.builder()
.name(viewName)
.isPublic(false)
.isSchemaPublic(false)
.query(query)
.build());
databaseService.createView(database, viewName, subset.getQuery());
final ViewDto view = databaseService.inspectView(database, viewName);
headers.set("Access-Control-Expose-Headers", "X-Id X-Headers");
headers.set("X-Headers", String.join(",", view.getColumns().stream().map(ViewColumnDto::getInternalName).toList()));
......@@ -439,16 +435,16 @@ public class SubsetEndpoint extends RestEndpoint {
mediaType = "application/json",
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<QueryDto> persist(@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("queryId") Long queryId,
public ResponseEntity<QueryDto> persist(@NotNull @PathVariable("databaseId") UUID databaseId,
@NotNull @PathVariable("queryId") UUID queryId,
@NotNull @Valid @RequestBody QueryPersistDto data,
@NotNull Principal principal) throws NotAllowedException,
RemoteUnavailableException, DatabaseNotFoundException, QueryStorePersistException,
DatabaseUnavailableException, QueryNotFoundException, UserNotFoundException, MetadataServiceException {
log.debug("endpoint persist query, databaseId={}, queryId={}, data.persist={}, principal.name={}", databaseId,
queryId, data.getPersist(), principal.getName());
final DatabaseDto database = credentialService.getDatabase(databaseId);
credentialService.getAccess(databaseId, getId(principal));
final DatabaseDto database = cacheService.getDatabase(databaseId);
cacheService.getAccess(databaseId, getId(principal));
try {
subsetService.persist(database, queryId, data.getPersist());
final QueryDto dto = subsetService.findById(database, queryId);
......
dbrepo-data-service/rest-service/src/main/resources/init/querystore.sql deleted 100644 → 0
View file @ 79a3df8a
CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\'\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \',\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END;
CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;
CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;
\ No newline at end of file