diff --git a/.docker/dist.tar.gz b/.docker/dist.tar.gz
index 41421af414e1f200cced2348ab00977bde12c8b1..411c92f64cd600841b98f1375c32b920895df2e0 100644
Binary files a/.docker/dist.tar.gz and b/.docker/dist.tar.gz differ
diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml
index ce5680133fcfe2690220850359b9f0dc5c28f3c8..54b7b8edbbe0652379fc74c5a81a58ada8a57e3a 100644
--- a/.docker/docker-compose.yml
+++ b/.docker/docker-compose.yml
@@ -275,6 +275,10 @@ services:
container_name: dbrepo-ui
hostname: ui
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.5
+ environment:
+ NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}"
+ NUXT_PUBLIC_API_SERVER: "${BASE_URL:-http://localhost}"
+ NUXT_PUBLIC_UPLOAD_CLIENT: "${BASE_URL:-http://localhost}/api/upload/files"
depends_on:
dbrepo-search-service:
condition: service_started
@@ -397,8 +401,8 @@ services:
- "-s3-endpoint=${STORAGE_ENDPOINT:-http://storage-service:9000}"
- "-s3-bucket=dbrepo"
environment:
- AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}"
- AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-seaweedfsadmin}"
+ AWS_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
+ AWS_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
AWS_REGION: "${STORAGE_REGION_NAME:-default}"
depends_on:
dbrepo-storage-service:
diff --git a/.docs/api/analyse-service.md b/.docs/api/analyse-service.md
index 3f5921968c7d1ebfd7cc4563c4b3fc4dccf7b23f..3529f0f3df3482ecefcadd17951c641c60c3422a 100644
--- a/.docs/api/analyse-service.md
+++ b/.docs/api/analyse-service.md
@@ -31,8 +31,11 @@ the [Storage Service](../storage-service), analysis for data types and primary k
with [`csv.Sniff().sniff(...)`](https://docs.python.org/3/library/csv.html#csv.Sniffer). This step is optional when
the separator was provided via HTTP-payload: `{"separator": ";", ...}`
3. With the separator known (either from step 2 or via HTTP-payload), the [`Pandas`](https://pypi.org/project/pandas/)
- guesses the headers and column types and enums, if the HTTP-payload contains `{"enum": true, ...}`. The data type
- is guessed by a combination of Pandas and heuristics.
+ guesses the headers and column types and enums by analysing the first 10.000 rows, if the HTTP-payload contains
+ `{"enum": true, ...}`. The data type is guessed by a combination of Pandas and heuristics.
+
+If your datasets are larger than 10.000 rows, increase the number of lines analysed by setting the `ANALYSE_NROWS`
+variable to the desired integer.
### Examples
diff --git a/.docs/api/gateway-service.md b/.docs/api/gateway-service.md
index 923b95a9f30ac9af06bb029682bd67bc7f2f0961..26ad76f092c599fbb865648577eba7b1a283ba72 100644
--- a/.docs/api/gateway-service.md
+++ b/.docs/api/gateway-service.md
@@ -39,6 +39,12 @@ services:
If your TLS private key as a password, you need to specify it in the `dbrepo.conf` file.
+### Connection Timeouts
+
+The reverse proxy has a defined timeout of 90 seconds on all requests (these are also enforced in the
+[User Interface](../ui) using the [`axios`](https://www.npmjs.com/package/axios) module). For large databases these
+timeouts may need to be increased, e.g. the timeout for creating subsets is by default already increased to 600 seconds.
+
### User Interface
To serve the [User Interface](../ui/) under different port than `80`, change the port mapping in
diff --git a/.docs/api/metadata-service.md b/.docs/api/metadata-service.md
index 039f2fa097ac3ba89d56ff6d735fc453bc3bc505..9d9c39ee25e62e14f610f5d84a1867af9a94e07f 100644
--- a/.docs/api/metadata-service.md
+++ b/.docs/api/metadata-service.md
@@ -23,9 +23,21 @@ types), semantic concepts (i.e. ontologies) and relational metadata (databases,
## Generation
-Most of the metadata available in DBRepo is generated automatically, leveraging the available information and taking
-the burden away from researchers, data stewards, etc. For example, the schema (names, constraints, data length) of
-generated tables and views is obtained from the `information_schema` database maintained by MariaDB internally.
+DBRepo generates metadata for managed tables automatically by querying MariaDB's internal structures
+(e.g. `information_schema`).
+
+!!! info "Managed Tables"
+
+ DBRepo only manages system-versioned tables, other tables are not supported. These other tables are ignored by
+ DBRepo and thus can co-exist in the same database. If you want a non-system-versioned table `my_table` to be managed
+ by DBRepo, make it system-versioned:
+
+ ```sql
+ ALTER TABLE `my_table` ADD SYSTEM VERSIONING;
+ ```
+
+ Then, refresh the managed table index by navigating to your database > Settings > Schema > Refresh. This action can
+ only be performed by the database owner.
## Identifiers
diff --git a/.docs/examples/health.md b/.docs/examples/health.md
new file mode 100644
index 0000000000000000000000000000000000000000..ebd2673c4423f604e0aa74d5a552e09212d00d4b
--- /dev/null
+++ b/.docs/examples/health.md
@@ -0,0 +1,25 @@
+---
+author: Martin Weise
+---
+
+## tl;dr
+
+[:fontawesome-solid-database: Dataset](https://dbrepo1.ec.tuwien.ac.at/database/32/info){ .md-button .md-button--primary target="_blank" }
+
+## Description
+
+This dataset contains anonymous behavioural and ocular recordings from healthy participants during performance of a
+decision-making task between action movements in which we studied the influence of social facilitation and social
+pressure. The original dataset is available on [Kaggle](https://doi.org/10.34740/kaggle/ds/4292829).
+
+## Solution
+
+Importing the original dataset from Kaggle into DBRepo allows for exploratory analysis and increased value for secondary
+use, e.g. with our [`pandas`](https://pypi.org/project/pandas/)-compatible [Python Library](../../api/python).
+
+## DBRepo Features
+
+- [x] System versioning
+- [x] Subset exploration
+- [x] Large dataset (3 tables, around 6GB)
+- [x] External data access for analysis
diff --git a/.docs/examples/theater.md b/.docs/examples/theater.md
new file mode 100644
index 0000000000000000000000000000000000000000..2ea453cfc9077f07490eada2c3d7c38f401053cf
--- /dev/null
+++ b/.docs/examples/theater.md
@@ -0,0 +1,27 @@
+---
+author: Martin Weise
+---
+
+## tl;dr
+
+[:fontawesome-solid-database: Dataset](https://dbrepo1.ec.tuwien.ac.at/pid/52){ .md-button .md-button--primary target="_blank" }
+
+## Description
+
+This section will be expanded soon.
+
+## Solution
+
+We host a mirror from the [public data](https://www.stadttheater.uni-hamburg.de/) provided by UHH.
+
+## DBRepo Features
+
+- [x] System versioning
+- [x] Subset exploration
+- [x] External data access for analysis
+
+## Acknowledgement
+
+This work was part of a cooperation with the [University of Hamburg](https://www.uni-hamburg.de/en.html).
+
+<img src="../../images/logos/uhh.png" width=100 />
\ No newline at end of file
diff --git a/.docs/images/logos/uhh.png b/.docs/images/logos/uhh.png
new file mode 100644
index 0000000000000000000000000000000000000000..dc3553771316f67d6912835b3b3d9763e35b114d
Binary files /dev/null and b/.docs/images/logos/uhh.png differ
diff --git a/.docs/installation.md b/.docs/installation.md
index 0404133900b41c28679e7a9562b8989783022742..9c864ead4686a1b4c417147ad564740aeb5b8c8e 100644
--- a/.docs/installation.md
+++ b/.docs/installation.md
@@ -47,10 +47,6 @@ curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-service
Call the helper script to regenerate the client secret of the `dbrepo-client` and set it as value of the
`AUTH_SERVICE_CLIENT_SECRET` variable in the `.env` file.
-```bash
-curl -sSL "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/.scripts/reg-client-secret.sh" | bash
-```
-
Update the rest of the default secrets in the `.env` file to secure passwords. You can use `openssl` for that, e.g.
`openssl rand -hex 16`. Set `auth_ldap.dn_lookup_bind.password` in `dist/rabbitmq.conf` to the value of
`SYSTEM_PASSWORD`.
@@ -69,7 +65,7 @@ Log into the Auth Service with the default credentials `admin` and the value of
:material-numeric-3-circle-outline:.
<figure markdown>
-{ .img-border }
+{ .img-border }
<figcaption>Figure 1: Select the Identity Service provider.</figcaption>
</figure>
@@ -78,9 +74,15 @@ but this is optional. Change the Bind credentials to the desired password :mater
the variable `IDENTITY_SERVICE_ADMIN_PASSWORD` in `.env`.
<figure markdown>
-{ .img-border }
+{ .img-border }
<figcaption>Figure 2: Update the Identity Service admin user credentials.</figcaption>
</figure>
+
+Update the client secret of the `dbrepo-client`:
+
+```bash
+curl -sSL "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.5/.scripts/reg-client-secret.sh" | bash
+```
Also, update the JWT key according to the
[Keycloak documentation](https://www.keycloak.org/docs/24.0.1/server_admin/index.html#rotating-keys). To secure your
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8322e26eb1201fe2c4429c6eca6cee85d74654b1..0bf1bc925538e7a74c85f2b6f12e290eaf8d10a0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -387,6 +387,7 @@ release-helm:
- "helm package ./helm/dbrepo --sign --key 'Martin Weise' --keyring ~/.gnupg/secring.gpg --destination ./build"
- "helm plugin install https://github.com/sigstore/helm-sigstore"
script:
+ - "helm push ./build/dbrepo-${CHART_VERSION}.tgz oci://${CI_REGISTRY2_URL}/helm"
- "helm sigstore upload ./build/dbrepo-${CHART_VERSION}.tgz"
release-docs:
diff --git a/dbrepo-analyse-service/app.py b/dbrepo-analyse-service/app.py
index 2dc9161746fbc52fb523b2bcfe934a2dac8ffbb4..def401c0e28e5c16c264d5fd280e5435fc44c45e 100644
--- a/dbrepo-analyse-service/app.py
+++ b/dbrepo-analyse-service/app.py
@@ -225,6 +225,7 @@ app.config["S3_ACCESS_KEY_ID"] = os.getenv('S3_ACCESS_KEY_ID', 'seaweedfsadmin')
app.config["S3_BUCKET"] = os.getenv('S3_BUCKET', 'dbrepo')
app.config["S3_ENDPOINT"] = os.getenv('S3_ENDPOINT', 'http://localhost:9000')
app.config["S3_SECRET_ACCESS_KEY"] = os.getenv('S3_SECRET_ACCESS_KEY', 'seaweedfsadmin')
+app.config["ANALYSE_NROWS"] = int(os.getenv('ANALYSE_NROWS', '10000'))
app.json_encoder = LazyJSONEncoder
diff --git a/dbrepo-analyse-service/determine_dt.py b/dbrepo-analyse-service/determine_dt.py
index a0890c2b7a9cd5a9e53649464cfa19ec47f0e45d..462f8b652b1e5c40c3a7cf0717c554c864052059 100644
--- a/dbrepo-analyse-service/determine_dt.py
+++ b/dbrepo-analyse-service/determine_dt.py
@@ -10,7 +10,7 @@ import pandas
from numpy import dtype, max, min
from flask import current_app
from pandas import DataFrame
-from pandas.errors import EmptyDataError
+from pandas.errors import EmptyDataError, ParserError
from api.dto import ColumnAnalysisDto, DataTypeDto, AnalysisDto
from clients.s3_client import S3Client
@@ -45,12 +45,14 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=',') ->
for encoding in ['utf-8', 'cp1252', 'latin1', 'iso-8859-1']:
try:
logging.debug(f"attempt parsing .csv using encoding {encoding}")
- df = pandas.read_csv(fh, delimiter=separator, nrows=100, lineterminator=line_terminator,
- index_col=False, encoding=encoding)
+ df = pandas.read_csv(fh, delimiter=separator, nrows=current_app.config['ANALYSE_NROWS'],
+ lineterminator=line_terminator, index_col=False, encoding=encoding)
logging.debug(f"parsing .csv using encoding {encoding} was successful")
break
+ except ParserError as error:
+ raise IOError(f"Failed to parse .csv using separator {separator}: {error}")
except (UnicodeDecodeError, EmptyDataError) as error:
- logging.warning(f"Failed to parse .csv using encoding {encoding}: {error}")
+ raise IOError(f"Failed to parse .csv using encoding {encoding}: {error}")
if df is None:
raise IOError(
f"Failed to parse .csv: no supported encoding found (one of: utf-8, cp1252, latin1, iso-8859-1)")
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
index 8f5f4a821484d23a51dd5c60a6bba658278fdb14..a7c74946923a36f639ae692e6d153881bcc2401f 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
@@ -86,19 +86,11 @@ public class SubsetEndpoint {
schema = @Schema(implementation = ApiErrorDto.class))}),
})
public ResponseEntity<List<QueryDto>> list(@NotNull @PathVariable("databaseId") Long databaseId,
- @RequestParam(name = "persisted", required = false) Boolean filterPersisted,
- Principal principal)
+ @RequestParam(name = "persisted", required = false) Boolean filterPersisted)
throws DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException,
QueryNotFoundException, NotAllowedException, MetadataServiceException {
log.debug("endpoint find subsets in database, databaseId={}, filterPersisted={}", databaseId, filterPersisted);
final PrivilegedDatabaseDto database = metadataServiceGateway.getDatabaseById(databaseId);
- if (!database.getIsPublic()) {
- if (principal == null) {
- log.error("Failed to find subsets in database: no access");
- throw new NotAllowedException("Failed to find subsets in database: no access");
- }
- metadataServiceGateway.getAccess(databaseId, UserUtil.getId(principal));
- }
final List<QueryDto> queries;
try {
queries = subsetService.findAll(database, filterPersisted);
@@ -151,8 +143,7 @@ public class SubsetEndpoint {
public ResponseEntity<?> findById(@NotNull @PathVariable("databaseId") Long databaseId,
@NotNull @PathVariable("subsetId") Long subsetId,
@NotNull HttpServletRequest httpServletRequest,
- @RequestParam(required = false) Instant timestamp,
- Principal principal)
+ @RequestParam(required = false) Instant timestamp)
throws DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException,
QueryNotFoundException, FormatNotAvailableException, StorageUnavailableException, QueryMalformedException,
SidecarExportException, StorageNotFoundException, NotAllowedException, UserNotFoundException,
@@ -161,13 +152,6 @@ public class SubsetEndpoint {
log.debug("endpoint find subset in database, databaseId={}, subsetId={}, accept={}, timestamp={}", databaseId,
subsetId, accept, timestamp);
final PrivilegedDatabaseDto database = metadataServiceGateway.getDatabaseById(databaseId);
- if (!database.getIsPublic()) {
- if (principal == null) {
- log.error("Failed to find subsets in database: no access");
- throw new NotAllowedException("Failed to find subsets in database: no access");
- }
- metadataServiceGateway.getAccess(databaseId, UserUtil.getId(principal));
- }
final QueryDto query;
try {
query = subsetService.findById(database, subsetId);
@@ -262,7 +246,7 @@ public class SubsetEndpoint {
StorageNotFoundException, QueryStoreInsertException, TableMalformedException, PaginationException,
QueryNotSupportedException, NotAllowedException, UserNotFoundException, MetadataServiceException {
log.debug("endpoint create subset in database, databaseId={}, data.statement={}, principal.name={}, " +
- "page={}, size={}, timestamp={}", databaseId, data.getStatement(), principal.getName(), page, size,
+ "page={}, size={}, timestamp={}", databaseId, data.getStatement(), principal.getName(), page, size,
timestamp);
/* check */
endpointValidator.validateDataParams(page, size);
@@ -360,21 +344,21 @@ public class SubsetEndpoint {
}
try {
final QueryDto query = subsetService.findById(database, subsetId);
- final Long count = subsetService.reExecuteCount(database, query);
final HttpHeaders headers = new HttpHeaders();
- headers.set("X-Count", "" + count);
headers.set("Access-Control-Expose-Headers", "X-Count");
- if (request.getMethod().equals("GET")) {
- final QueryResultDto result = subsetService.reExecute(database, query, page, size, null, null);
- result.setId(subsetId);
- log.trace("re-execute query resulted in result {}", result);
+ if (request.getMethod().equals("HEAD")) {
+ final Long count = subsetService.reExecuteCount(database, query);
+ headers.set("X-Count", "" + count);
return ResponseEntity.ok()
.headers(headers)
- .body(result);
+ .build();
}
+ final QueryResultDto result = subsetService.reExecute(database, query, page, size, null, null);
+ result.setId(subsetId);
+ log.trace("re-execute query resulted in result {}", result);
return ResponseEntity.ok()
.headers(headers)
- .build();
+ .body(result);
} catch (SQLException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
index f4d31bbc216b333656d1ac70d0e3f28ddd49be0f..e26a37b43aaa1b5307cf87d294e1dedc5cef87c4 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
@@ -25,6 +25,7 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
+import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
@@ -32,6 +33,7 @@ import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
@@ -189,6 +191,7 @@ public class TableEndpoint {
@RequestParam(required = false) Instant timestamp,
@RequestParam(required = false) Long page,
@RequestParam(required = false) Long size,
+ @NotNull HttpServletRequest request,
Principal principal)
throws DatabaseUnavailableException, RemoteUnavailableException, TableNotFoundException,
TableMalformedException, PaginationException, QueryMalformedException, MetadataServiceException,
@@ -221,7 +224,12 @@ public class TableEndpoint {
final HttpHeaders headers = new HttpHeaders();
headers.set("Access-Control-Expose-Headers", "X-Count");
try {
- headers.set("X-Count", "" + tableService.getCount(table, timestamp));
+ if (request.getMethod().equals("HEAD")) {
+ headers.set("X-Count", "" + tableService.getCount(table, timestamp));
+ return ResponseEntity.ok()
+ .headers(headers)
+ .build();
+ }
final QueryResultDto dto = tableService.getData(table, timestamp, page, size);
return ResponseEntity.ok()
.headers(headers)
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
index e9dbfd5c3a7f3d59088f048398cd2c4f9d9d8f45..e4ddab603685103027dcb6dfad936b882c987294 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
@@ -92,8 +92,7 @@ public class ViewEndpoint {
})
public ResponseEntity<List<ViewDto>> getSchema(@NotBlank @PathVariable("databaseId") Long databaseId)
throws DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException,
- ViewMalformedException, ViewNotFoundException, DatabaseMalformedException, ViewSchemaException,
- MetadataServiceException {
+ ViewNotFoundException, DatabaseMalformedException, MetadataServiceException {
log.debug("endpoint inspect view schemas, databaseId={}", databaseId);
final PrivilegedDatabaseDto database = metadataServiceGateway.getDatabaseById(databaseId);
try {
@@ -265,20 +264,19 @@ public class ViewEndpoint {
metadataServiceGateway.getAccess(databaseId, UserUtil.getId(principal));
}
try {
- final Long count = viewService.count(view, timestamp);
final HttpHeaders headers = new HttpHeaders();
- headers.set("X-Count", "" + count);
headers.set("Access-Control-Expose-Headers", "X-Count");
- if (request.getMethod().equals("GET")) {
- final QueryResultDto result = viewService.data(view, timestamp, page, size);
- log.trace("get view data resulted in result {}", result);
+ if (request.getMethod().equals("HEAD")) {
+ headers.set("X-Count", "" + viewService.count(view, timestamp));
return ResponseEntity.ok()
.headers(headers)
- .body(result);
+ .build();
}
+ final QueryResultDto result = viewService.data(view, timestamp, page, size);
+ log.trace("get view data resulted in result {}", result);
return ResponseEntity.ok()
.headers(headers)
- .build();
+ .body(result);
} catch (SQLException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
index d554667467009b396caee8f2800b522f21f8ace6..6a0015b6f61e567fbf65f2e8f97f6c568fe142fa 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java
@@ -29,7 +29,6 @@ import org.springframework.security.test.context.support.WithMockUser;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.io.InputStream;
-import java.security.Principal;
import java.sql.SQLException;
import java.time.Instant;
import java.util.List;
@@ -69,7 +68,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
DatabaseNotFoundException, RemoteUnavailableException, SQLException, MetadataServiceException {
/* test */
- final List<QueryDto> response = generic_findAllById(DATABASE_3_ID, DATABASE_3_PRIVILEGED_DTO, null);
+ final List<QueryDto> response = generic_findAllById(DATABASE_3_ID, DATABASE_3_PRIVILEGED_DTO);
assertEquals(6, response.size());
}
@@ -79,17 +78,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* test */
assertThrows(DatabaseNotFoundException.class, () -> {
- generic_findAllById(null, null, null);
- });
- }
-
- @Test
- @WithAnonymousUser
- public void findAllById_privateNoAccess_fails() {
-
- /* test */
- assertThrows(NotAllowedException.class, () -> {
- generic_findAllById(DATABASE_1_ID, DATABASE_1_PRIVILEGED_DTO, null);
+ generic_findAllById(null, null);
});
}
@@ -105,7 +94,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
/* test */
- generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.APPLICATION_JSON, null, null);
+ generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.APPLICATION_JSON, null);
}
@Test
@@ -126,7 +115,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(mock);
/* test */
- generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.parseMediaType("text/csv"), null, null);
+ generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.parseMediaType("text/csv"), null);
}
@Test
@@ -147,7 +136,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(mock);
/* test */
- generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.parseMediaType("text/csv"), Instant.now(), null);
+ generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.parseMediaType("text/csv"), Instant.now());
}
@Test
@@ -161,7 +150,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* test */
assertThrows(DatabaseNotFoundException.class, () -> {
- generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.APPLICATION_JSON, null, null);
+ generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.APPLICATION_JSON, null);
});
}
@@ -272,9 +261,6 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* test */
final ResponseEntity<QueryResultDto> response = subsetEndpoint.getData(DATABASE_3_ID, QUERY_5_ID, null, httpServletRequest, null, null);
assertEquals(HttpStatus.OK, response.getStatusCode());
- assertNotNull(response.getHeaders().get("X-Count"));
- assertEquals(1, response.getHeaders().get("X-Count").size());
- assertEquals(QUERY_5_RESULT_NUMBER, Long.parseLong(response.getHeaders().get("X-Count").get(0)));
assertNotNull(response.getHeaders().get("Access-Control-Expose-Headers"));
assertEquals(1, response.getHeaders().get("Access-Control-Expose-Headers").size());
assertEquals("X-Count", response.getHeaders().get("Access-Control-Expose-Headers").get(0));
@@ -325,9 +311,6 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* test */
final ResponseEntity<QueryResultDto> response = subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, null);
assertEquals(HttpStatus.OK, response.getStatusCode());
- assertNotNull(response.getHeaders().get("X-Count"));
- assertEquals(1, response.getHeaders().get("X-Count").size());
- assertEquals(QUERY_1_RESULT_NUMBER, Long.parseLong(response.getHeaders().get("X-Count").get(0)));
assertNotNull(response.getBody());
}
@@ -464,7 +447,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
});
}
- protected List<QueryDto> generic_findAllById(Long databaseId, PrivilegedDatabaseDto database, Principal principal)
+ protected List<QueryDto> generic_findAllById(Long databaseId, PrivilegedDatabaseDto database)
throws DatabaseUnavailableException, NotAllowedException, QueryNotFoundException, DatabaseNotFoundException,
RemoteUnavailableException, SQLException, MetadataServiceException {
@@ -481,16 +464,16 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
}
/* test */
- final ResponseEntity<List<QueryDto>> response = subsetEndpoint.list(databaseId, null, principal);
+ final ResponseEntity<List<QueryDto>> response = subsetEndpoint.list(databaseId, null);
assertEquals(HttpStatus.OK, response.getStatusCode());
return response.getBody();
}
- protected void generic_findById(Long subsetId, QueryDto subset, MediaType accept, Instant timestamp,
- Principal principal) throws UserNotFoundException, DatabaseUnavailableException,
- StorageUnavailableException, NotAllowedException, QueryMalformedException, QueryNotFoundException,
- DatabaseNotFoundException, SidecarExportException, RemoteUnavailableException, FormatNotAvailableException,
- StorageNotFoundException, SQLException, MetadataServiceException {
+ protected void generic_findById(Long subsetId, QueryDto subset, MediaType accept, Instant timestamp)
+ throws UserNotFoundException, DatabaseUnavailableException, StorageUnavailableException,
+ NotAllowedException, QueryMalformedException, QueryNotFoundException, DatabaseNotFoundException,
+ SidecarExportException, RemoteUnavailableException, FormatNotAvailableException, StorageNotFoundException,
+ SQLException, MetadataServiceException {
/* mock */
when(queryService.findById(DATABASE_3_PRIVILEGED_DTO, subsetId))
@@ -499,7 +482,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.thenReturn(accept.toString());
/* test */
- final ResponseEntity<?> response = subsetEndpoint.findById(DATABASE_3_ID, subsetId, mockHttpServletRequest, timestamp, principal);
+ final ResponseEntity<?> response = subsetEndpoint.findById(DATABASE_3_ID, subsetId, mockHttpServletRequest, timestamp);
assertEquals(HttpStatus.OK, response.getStatusCode());
assertNotNull(response.getBody());
}
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java
index 29a33d1d9cbb3dd95a5640610d7620a36d1800db..76c34ef47cd2d13f698f3d552b9aebbbb4175492 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java
@@ -9,6 +9,7 @@ import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
import at.tuwien.service.TableService;
import at.tuwien.test.AbstractUnitTest;
+import jakarta.servlet.http.HttpServletRequest;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -40,6 +41,9 @@ public class TableEndpointUnitTest extends AbstractUnitTest {
@Autowired
private TableEndpoint tableEndpoint;
+ @Autowired
+ private HttpServletRequest httpServletRequest;
+
@MockBean
private TableService tableService;
@@ -152,11 +156,8 @@ public class TableEndpointUnitTest extends AbstractUnitTest {
.thenReturn(TABLE_8_DATA_DTO);
/* test */
- final ResponseEntity<QueryResultDto> response = tableEndpoint.getData(DATABASE_3_ID, TABLE_8_ID, null, null, null, null);
+ final ResponseEntity<QueryResultDto> response = tableEndpoint.getData(DATABASE_3_ID, TABLE_8_ID, null, null, null, httpServletRequest, null);
assertEquals(HttpStatus.OK, response.getStatusCode());
- assertNotNull(response.getHeaders().get("X-Count"));
- assertEquals(1, response.getHeaders().get("X-Count").size());
- assertEquals(QUERY_5_RESULT_NUMBER, Long.parseLong(response.getHeaders().get("X-Count").get(0)));
assertNotNull(response.getHeaders().get("Access-Control-Expose-Headers"));
assertEquals(1, response.getHeaders().get("Access-Control-Expose-Headers").size());
assertEquals("X-Count", response.getHeaders().get("Access-Control-Expose-Headers").get(0));
@@ -175,7 +176,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest {
/* test */
assertThrows(TableNotFoundException.class, () -> {
- tableEndpoint.getData(DATABASE_3_ID, TABLE_8_ID, null, null, null, null);
+ tableEndpoint.getData(DATABASE_3_ID, TABLE_8_ID, null, null, null, httpServletRequest, null);
});
}
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java
index b9b814378e9bd537d42c421c3e3fab81d381443f..543b4a5cf299d48a8a4594f95e8586752d4278cc 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java
@@ -161,17 +161,12 @@ public class ViewEndpointUnitTest extends AbstractUnitTest {
.thenReturn(DATABASE_1_USER_1_READ_ACCESS_DTO);
when(httpServletRequest.getMethod())
.thenReturn("GET");
- when(viewService.count(eq(VIEW_1_PRIVILEGED_DTO), any(Instant.class)))
- .thenReturn(VIEW_1_DATA_COUNT);
when(viewService.data(eq(VIEW_1_PRIVILEGED_DTO), any(Instant.class), eq(0L), eq(10L)))
.thenReturn(VIEW_1_DATA_DTO);
/* test */
final ResponseEntity<QueryResultDto> response = viewEndpoint.getData(DATABASE_1_ID, VIEW_1_ID, null, null, null, httpServletRequest, USER_1_PRINCIPAL);
assertEquals(HttpStatus.OK, response.getStatusCode());
- assertNotNull(response.getHeaders().get("X-Count"));
- assertEquals(1, response.getHeaders().get("X-Count").size());
- assertEquals(VIEW_1_DATA_COUNT, Long.parseLong(response.getHeaders().get("X-Count").get(0)));
assertNotNull(response.getHeaders().get("Access-Control-Expose-Headers"));
assertEquals(1, response.getHeaders().get("Access-Control-Expose-Headers").size());
assertEquals("X-Count", response.getHeaders().get("Access-Control-Expose-Headers").get(0));
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
index 9d42c1a54df551749416b0b1fcc5c6c56abed3f8..5898372633b0d63c5b256f6b567f8a8869583ab7 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
@@ -126,12 +126,12 @@ public class PrometheusEndpointMvcTest extends AbstractUnitTest {
}
@Test
- @WithMockUser(username = USER_1_USERNAME, authorities = {"dbrepo_subset_list", "execute-query", "persist-query"})
+ @WithMockUser(username = USER_1_USERNAME, authorities = {"execute-query", "persist-query"})
public void prometheusSubsetEndpoint_succeeds() {
/* mock */
try {
- subsetEndpoint.list(DATABASE_1_ID, null, USER_1_PRINCIPAL);
+ subsetEndpoint.list(DATABASE_1_ID, null);
} catch (Exception e) {
/* ignore */
}
@@ -151,7 +151,7 @@ public class PrometheusEndpointMvcTest extends AbstractUnitTest {
/* ignore */
}
try {
- subsetEndpoint.findById(DATABASE_1_ID, QUERY_1_ID, new MockHttpServletRequest(), null, USER_1_PRINCIPAL);
+ subsetEndpoint.findById(DATABASE_1_ID, QUERY_1_ID, new MockHttpServletRequest(), null);
} catch (Exception e) {
/* ignore */
}
@@ -171,7 +171,7 @@ public class PrometheusEndpointMvcTest extends AbstractUnitTest {
/* mock */
try {
- tableEndpoint.getData(DATABASE_1_ID, TABLE_1_ID, null, null, null, null);
+ tableEndpoint.getData(DATABASE_1_ID, TABLE_1_ID, null, null, null, httpServletRequest, null);
} catch (Exception e) {
/* ignore */
}
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java
index be1f6b5dae37db17ec5e90ad001a54ff77869886..540e17850a535b8776daf4a6a7bf53a5499f1838 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java
@@ -332,12 +332,39 @@ public class SchemaServiceIntegrationTest extends AbstractUnitTest {
assertEquals("other_id", pk1.getColumn().getName());
assertEquals("other_id", pk1.getColumn().getInternalName());
assertEquals(ColumnTypeDto.BIGINT, pk1.getColumn().getColumnType());
+ }
+
+ @Test
+ public void inspectTable_exoticBoolean_succeeds() throws TableNotFoundException, SQLException {
+ /* test */
+ final TableDto response = schemaService.inspectTable(DATABASE_1_PRIVILEGED_DTO, "exotic_boolean");
+ final ConstraintsDto constraints = response.getConstraints();
+ final List<PrimaryKeyDto> primaryKey = new LinkedList<>(constraints.getPrimaryKey());
+ assertEquals(1, primaryKey.size());
+ final PrimaryKeyDto pk0 = primaryKey.get(0);
+ assertNull(pk0.getId());
+ assertNotNull(pk0.getTable());
+ assertNull(pk0.getTable().getId());
+ assertEquals("exotic_boolean", pk0.getTable().getName());
+ assertEquals("exotic_boolean", pk0.getTable().getInternalName());
+ assertNotNull(pk0.getColumn());
+ assertNull(pk0.getColumn().getId());
+ assertNull(pk0.getColumn().getTableId());
+ assertEquals(DATABASE_1_ID, pk0.getColumn().getDatabaseId());
+ assertNull(pk0.getColumn().getAlias());
+ assertEquals("bool_default", pk0.getColumn().getName());
+ assertEquals("bool_default", pk0.getColumn().getInternalName());
+ assertEquals(ColumnTypeDto.BOOL, pk0.getColumn().getColumnType());
+ final List<ColumnDto> columns = response.getColumns();
+ assertEquals(3, columns.size());
+ assertColumn(columns.get(0), null, null, DATABASE_1_ID, "bool_default", "bool_default", ColumnTypeDto.BOOL, null, 0L, false, null, null);
+ assertColumn(columns.get(1), null, null, DATABASE_1_ID, "bool_tinyint", "bool_tinyint", ColumnTypeDto.BOOL, null, 0L, false, null, null);
+ assertColumn(columns.get(2), null, null, DATABASE_1_ID, "bool_tinyint_unsigned", "bool_tinyint_unsigned", ColumnTypeDto.BOOL, null, 0L, false, null, null);
}
@Test
- public void inspectView_succeeds() throws ViewMalformedException, SQLException, ViewNotFoundException,
- ViewSchemaException {
+ public void inspectView_succeeds() throws SQLException, ViewNotFoundException {
/* test */
final ViewDto response = schemaService.inspectView(DATABASE_1_PRIVILEGED_DTO, "not_in_metadata_db2");
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java
index 41fa4963fb26158f11060db383c4cb973e66e1c0..b8267e959e32f00545ec6670af98b420efb17c13 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java
@@ -334,7 +334,7 @@ public class TableServiceIntegrationTest extends AbstractUnitTest {
/* test */
final List<TableDto> response = tableService.getSchemas(DATABASE_1_PRIVILEGED_DTO);
- assertEquals(3, response.size());
+ assertEquals(4, response.size());
final TableDto table0 = response.get(0);
Assertions.assertEquals("complex_foreign_keys", table0.getInternalName());
Assertions.assertEquals("complex_foreign_keys", table0.getName());
@@ -411,32 +411,53 @@ public class TableServiceIntegrationTest extends AbstractUnitTest {
assertEquals(0, constraints1.getUniques().size());
/* table 2 */
final TableDto table2 = response.get(2);
- Assertions.assertEquals("not_in_metadata_db", table2.getInternalName());
- Assertions.assertEquals("not_in_metadata_db", table2.getName());
+ Assertions.assertEquals("exotic_boolean", table2.getInternalName());
+ Assertions.assertEquals("exotic_boolean", table2.getName());
Assertions.assertEquals(DATABASE_1_ID, table2.getTdbid());
assertTrue(table2.getIsVersioned());
Assertions.assertEquals(DATABASE_1_PUBLIC, table2.getIsPublic());
final List<ColumnDto> columns2 = table2.getColumns();
assertNotNull(columns2);
- Assertions.assertEquals(5, columns2.size());
- assertColumn(columns2.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null);
- assertColumn(columns2.get(1), null, null, DATABASE_1_ID, "given_name", "given_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null);
- assertColumn(columns2.get(2), null, null, DATABASE_1_ID, "middle_name", "middle_name", ColumnTypeDto.VARCHAR, 255L, null, true, null, null);
- assertColumn(columns2.get(3), null, null, DATABASE_1_ID, "family_name", "family_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null);
- assertColumn(columns2.get(4), null, null, DATABASE_1_ID, "age", "age", ColumnTypeDto.INT, 10L, 0L, false, null, null);
+ Assertions.assertEquals(3, columns2.size());
+ assertColumn(columns2.get(0), null, null, DATABASE_1_ID, "bool_default", "bool_default", ColumnTypeDto.BOOL, null, 0L, false, null, null);
+ assertColumn(columns2.get(1), null, null, DATABASE_1_ID, "bool_tinyint", "bool_tinyint", ColumnTypeDto.BOOL, null, 0L, false, null, null);
+ assertColumn(columns2.get(2), null, null, DATABASE_1_ID, "bool_tinyint_unsigned", "bool_tinyint_unsigned", ColumnTypeDto.BOOL, null, 0L, false, null, null);
final ConstraintsDto constraints2 = table2.getConstraints();
assertNotNull(constraints2);
final Set<PrimaryKeyDto> primaryKey2 = constraints2.getPrimaryKey();
Assertions.assertEquals(1, primaryKey2.size());
final Set<String> checks2 = constraints2.getChecks();
- Assertions.assertEquals(1, checks2.size());
- Assertions.assertEquals(Set.of("`age` > 0 and `age` < 120"), checks2);
+ Assertions.assertEquals(0, checks2.size());
final List<UniqueDto> uniques2 = constraints2.getUniques();
- Assertions.assertEquals(1, uniques2.size());
- Assertions.assertEquals(2, uniques2.get(0).getColumns().size());
- Assertions.assertEquals("not_in_metadata_db", uniques2.get(0).getTable().getInternalName());
- Assertions.assertEquals("given_name", uniques2.get(0).getColumns().get(0).getInternalName());
- Assertions.assertEquals("family_name", uniques2.get(0).getColumns().get(1).getInternalName());
+ Assertions.assertEquals(0, uniques2.size());
+ /* table 3 */
+ final TableDto table3 = response.get(3);
+ Assertions.assertEquals("not_in_metadata_db", table3.getInternalName());
+ Assertions.assertEquals("not_in_metadata_db", table3.getName());
+ Assertions.assertEquals(DATABASE_1_ID, table3.getTdbid());
+ assertTrue(table3.getIsVersioned());
+ Assertions.assertEquals(DATABASE_1_PUBLIC, table3.getIsPublic());
+ final List<ColumnDto> columns3 = table3.getColumns();
+ assertNotNull(columns3);
+ Assertions.assertEquals(5, columns3.size());
+ assertColumn(columns3.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null);
+ assertColumn(columns3.get(1), null, null, DATABASE_1_ID, "given_name", "given_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null);
+ assertColumn(columns3.get(2), null, null, DATABASE_1_ID, "middle_name", "middle_name", ColumnTypeDto.VARCHAR, 255L, null, true, null, null);
+ assertColumn(columns3.get(3), null, null, DATABASE_1_ID, "family_name", "family_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null);
+ assertColumn(columns3.get(4), null, null, DATABASE_1_ID, "age", "age", ColumnTypeDto.INT, 10L, 0L, false, null, null);
+ final ConstraintsDto constraints3 = table3.getConstraints();
+ assertNotNull(constraints3);
+ final Set<PrimaryKeyDto> primaryKey3 = constraints3.getPrimaryKey();
+ Assertions.assertEquals(1, primaryKey3.size());
+ final Set<String> checks3 = constraints3.getChecks();
+ Assertions.assertEquals(1, checks3.size());
+ Assertions.assertEquals(Set.of("`age` > 0 and `age` < 120"), checks3);
+ final List<UniqueDto> uniques3 = constraints3.getUniques();
+ Assertions.assertEquals(1, uniques3.size());
+ Assertions.assertEquals(2, uniques3.get(0).getColumns().size());
+ Assertions.assertEquals("not_in_metadata_db", uniques3.get(0).getTable().getInternalName());
+ Assertions.assertEquals("given_name", uniques3.get(0).getColumns().get(0).getInternalName());
+ Assertions.assertEquals("family_name", uniques3.get(0).getColumns().get(1).getInternalName());
}
@Test
diff --git a/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql b/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql
index 79755c97bf68fd822869277e3ad867c582f4e68c..7c3ca99ce39f61755752d2ecbb607ad6ed86e386 100644
--- a/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql
+++ b/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql
@@ -32,7 +32,7 @@ CREATE TABLE complex_foreign_keys
(
id BIGINT NOT NULL PRIMARY KEY,
weather_id BIGINT NOT NULL,
- other_id BIGINT NOT NULL,
+ other_id BIGINT NOT NULL,
FOREIGN KEY (weather_id, other_id) REFERENCES complex_primary_key (id, `other_id`)
) WITH SYSTEM VERSIONING;
@@ -42,6 +42,13 @@ CREATE TABLE sensor
`value` DECIMAL
) WITH SYSTEM VERSIONING;
+CREATE TABLE exotic_boolean
+(
+ `bool_default` BOOLEAN NOT NULL PRIMARY KEY,
+ `bool_tinyint` TINYINT(1) NOT NULL,
+ `bool_tinyint_unsigned` TINYINT(1) UNSIGNED NOT NULL
+) WITH SYSTEM VERSIONING;
+
INSERT INTO weather_location (location, lat, lng)
VALUES ('Albury', -36.0653583, 146.9112214),
('Sydney', -33.847927, 150.6517942),
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java
index 6e233395d92ae03785ba820055cb7d421cfc7b38..1d73f6219a77e05f33166e23f75e16cafebc3e6f 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java
@@ -58,20 +58,20 @@ public class KeycloakGatewayImpl implements KeycloakGateway {
.exchange(url, HttpMethod.POST, new HttpEntity<>(payload, headers), TokenDto.class);
} catch (HttpServerErrorException e) {
log.error("Failed to obtain user token: {}", e.getMessage());
- throw new AuthServiceConnectionException("Service unavailable", e);
+ throw new AuthServiceConnectionException("Failed to obtain user token: " + e.getMessage(), e);
} catch (HttpClientErrorException.BadRequest e) {
if (e.getResponseBodyAsByteArray() != null && e.getResponseBodyAsByteArray().length > 0) {
final KeycloakErrorDto error = e.getResponseBodyAs(KeycloakErrorDto.class);
if (error != null && error.getError().equals("invalid_grant")) {
log.error("Failed to obtain user token: {}", error.getErrorDescription());
- throw new AccountNotSetupException(error.getErrorDescription());
+ throw new AccountNotSetupException("Failed to obtain user token: " + error.getErrorDescription(), e);
}
}
log.error("Failed to obtain user token: bad request");
throw new CredentialsInvalidException("Bad request", e);
} catch (HttpClientErrorException.Unauthorized e) {
log.error("Failed to obtain user token: invalid credentials");
- throw new CredentialsInvalidException("Invalid credentials", e);
+ throw new CredentialsInvalidException("Invalid credentials: " + e.getMessage(), e);
}
return response.getBody();
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java
index 06641b738b91c4798e9a57b26eb011fbc6c3c4cd..b4cb2ff5043c46e1e5ece49a45690ef04782bd3c 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java
@@ -7,7 +7,6 @@ import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.internal.PrivilegedViewDto;
import at.tuwien.api.database.table.TableDto;
-import at.tuwien.api.database.table.TableStatisticDto;
import at.tuwien.api.database.table.internal.PrivilegedTableDto;
import at.tuwien.api.identifier.IdentifierDto;
import at.tuwien.api.user.PrivilegedUserDto;
@@ -56,7 +55,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway {
throw new RemoteUnavailableException("Failed to find container: " + e.getMessage(), e);
} catch (HttpClientErrorException.NotFound e) {
log.error("Failed to find container with id {}: {}", containerId, e.getMessage());
- throw new ContainerNotFoundException("Failed to find container: " + e.getMessage());
+ throw new ContainerNotFoundException("Failed to find container: " + e.getMessage(), e);
}
if (response.getStatusCode() != HttpStatus.OK) {
log.error("Failed to find container with id {}: service responded unsuccessful: {}", containerId, response.getStatusCode());
@@ -88,7 +87,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway {
throw new RemoteUnavailableException("Failed to find database: " + e.getMessage(), e);
} catch (HttpClientErrorException.NotFound e) {
log.error("Failed to find database with id {}: body is null", id);
- throw new DatabaseNotFoundException("Failed to find database: body is null", e);
+ throw new DatabaseNotFoundException("Failed to find database: body is null: " + e.getMessage(), e);
}
if (response.getStatusCode() != HttpStatus.OK) {
log.error("Failed to find database with id {}: service responded unsuccessful: {}", id, response.getStatusCode());
@@ -141,7 +140,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway {
throw new RemoteUnavailableException("Failed to find table: " + e.getMessage(), e);
} catch (HttpClientErrorException.NotFound e) {
log.error("Failed to find table with id {}: not found: {}", id, e.getMessage());
- throw new TableNotFoundException("Failed to find table: " + e.getMessage());
+ throw new TableNotFoundException("Failed to find table: " + e.getMessage(), e);
}
if (!response.getStatusCode().equals(HttpStatus.OK)) {
log.error("Failed to find table with id {}: service responded unsuccessful: {}", id, response.getStatusCode());
@@ -179,7 +178,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway {
throw new RemoteUnavailableException("Failed to find view: " + e.getMessage(), e);
} catch (HttpClientErrorException.NotFound e) {
log.error("Failed to find view with id {}: not found: {}", id, e.getMessage());
- throw new ViewNotFoundException("Failed to find view: " + e.getMessage());
+ throw new ViewNotFoundException("Failed to find view: " + e.getMessage(), e);
}
if (!response.getStatusCode().equals(HttpStatus.OK)) {
log.error("Failed to find view with id {}: service responded unsuccessful: {}", id, response.getStatusCode());
@@ -214,7 +213,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway {
throw new RemoteUnavailableException("Failed to find user: " + e.getMessage(), e);
} catch (HttpClientErrorException.NotFound e) {
log.error("Failed to find user with id {}: not found: {}", userId, e.getMessage());
- throw new UserNotFoundException("Failed to find user: " + e.getMessage());
+ throw new UserNotFoundException("Failed to find user: " + e.getMessage(), e);
}
if (!response.getStatusCode().equals(HttpStatus.OK)) {
log.error("Failed to find user with id {}: service responded unsuccessful: {}", userId, response.getStatusCode());
@@ -238,7 +237,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway {
throw new RemoteUnavailableException("Failed to find user: " + e.getMessage(), e);
} catch (HttpClientErrorException.NotFound e) {
log.error("Failed to find user with id {}: not found: {}", userId, e.getMessage());
- throw new UserNotFoundException("Failed to find user: " + e.getMessage());
+ throw new UserNotFoundException("Failed to find user: " + e.getMessage(), e);
}
if (!response.getStatusCode().equals(HttpStatus.OK)) {
log.error("Failed to find user with id {}: service responded unsuccessful: {}", userId, response.getStatusCode());
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java
index 796ef3edebc452471569dd6f04c81bb4731891cd..b34053ec88a47a1f02ab99048fffc7e7f0cf35dd 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java
@@ -21,6 +21,7 @@ import at.tuwien.api.database.table.constraints.foreign.ForeignKeyReferenceDto;
import at.tuwien.api.database.table.constraints.foreign.ReferenceTypeDto;
import at.tuwien.api.database.table.constraints.primary.PrimaryKeyDto;
import at.tuwien.api.database.table.constraints.unique.UniqueDto;
+import at.tuwien.api.user.UserDto;
import at.tuwien.config.QueryConfig;
import at.tuwien.exception.QueryNotFoundException;
import at.tuwien.exception.TableNotFoundException;
@@ -97,7 +98,8 @@ public interface DataMapper {
/**
* Map the inspected schema to either an existing view/table and append e.g. column or (if not existing) create a new view/table.
- * @param database The database.
+ *
+ * @param database The database.
* @param resultSet The inspected schema.
* @return The database containing the updated view/table.
* @throws SQLException
@@ -152,31 +154,23 @@ public interface DataMapper {
.databaseId(table.getTdbid())
.description(resultSet.getString(11))
.build();
+ final String dataType = resultSet.getString(8);
if (column.getColumnType().equals(ColumnTypeDto.ENUM)) {
- column.setEnums(Arrays.stream(resultSet.getString(8)
- .substring(0, resultSet.getString(8).length() - 1)
+ column.setEnums(Arrays.stream(dataType.substring(0, resultSet.getString(8).length() - 1)
.replace("enum(", "")
.split(","))
.map(value -> value.replace("'", ""))
.toList());
}
if (column.getColumnType().equals(ColumnTypeDto.SET)) {
- column.setSets(Arrays.stream(resultSet.getString(8)
- .substring(0, resultSet.getString(8).length() - 1)
+ column.setSets(Arrays.stream(dataType.substring(0, dataType.length() - 1)
.replace("set(", "")
.split(","))
.map(value -> value.replace("'", ""))
.toList());
}
- /* constraints */
- if (resultSet.getString(9) != null && resultSet.getString(9).equals("PRI")) {
- table.getConstraints().getPrimaryKey().add(PrimaryKeyDto.builder()
- .table(tableDtoToTableBriefDto(table))
- .column(columnDtoToColumnBriefDto(column))
- .build());
- }
/* fix boolean and set size for others */
- if (resultSet.getString(8).equalsIgnoreCase("tinyint(1)")) {
+ if (dataType.startsWith("tinyint(1)")) {
column.setColumnType(ColumnTypeDto.BOOL);
} else if (resultSet.getString(5) != null) {
column.setSize(resultSet.getLong(5));
@@ -196,6 +190,13 @@ public interface DataMapper {
.id(queryConfig.getDefaultTimeFormatId())
.build());
}
+ /* constraints */
+ if (resultSet.getString(9) != null && resultSet.getString(9).equals("PRI")) {
+ table.getConstraints().getPrimaryKey().add(PrimaryKeyDto.builder()
+ .table(tableDtoToTableBriefDto(table))
+ .column(columnDtoToColumnBriefDto(column))
+ .build());
+ }
table.getColumns()
.add(column);
return table;
@@ -241,8 +242,9 @@ public interface DataMapper {
/**
* Parse columns from a SQL statement of a known database.
+ *
* @param database The database.
- * @param query The SQL statement.
+ * @param query The SQL statement.
* @return The list of columns.
* @throws JSQLParserException The table/view or column was not found in the database.
*/
@@ -401,6 +403,9 @@ public interface DataMapper {
.created(LocalDateTime.parse(data.getString(2), mariaDbFormatter)
.atZone(ZoneId.of("UTC"))
.toInstant())
+ .creator(UserDto.builder()
+ .id(UUID.fromString(data.getString(3)))
+ .build())
.createdBy(UUID.fromString(data.getString(3)))
.query(data.getString(4))
.queryHash(data.getString(5))
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
index 74e9ffe66d7428d5fdfb7d94909794f034cfd045..a88cdb5078b9a6fe348b082ff089de01a444bc61 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
@@ -73,6 +73,30 @@ public interface MariaDbMapper {
return statement.toString();
}
+ default String databaseRevokePrivilegesQuery(String username) {
+ final StringBuilder statement = new StringBuilder("REVOKE ALL PRIVILEGES ON *.* FROM `")
+ .append(username)
+ .append("`@`%`;");
+ log.trace("mapped revoke privileges statement: {}", statement);
+ return statement.toString();
+ }
+
+ default String databaseGrantProcedureQuery(String username, String procedure) {
+ final StringBuilder statement = new StringBuilder("GRANT EXECUTE ON PROCEDURE `")
+ .append(procedure)
+ .append("` TO `")
+ .append(username)
+ .append("`@`%`;");
+ log.trace("mapped revoke privileges statement: {}", statement);
+ return statement.toString();
+ }
+
+ default String databaseFlushPrivilegesQuery() {
+ final String statement = "FLUSH PRIVILEGES;";
+ log.trace("mapped flush privileges statement: {}", statement);
+ return statement;
+ }
+
@Named("createDatabase")
default String databaseCreateDatabaseQuery(String database) {
final StringBuilder statement = new StringBuilder("CREATE DATABASE `")
@@ -82,6 +106,60 @@ public interface MariaDbMapper {
return statement.toString();
}
+ default String queryStoreCreateSequenceRawQuery() {
+ final String statement = "CREATE SEQUENCE `qs_queries_seq` NOCACHE;";
+ log.trace("mapped create query store sequence statement: {}", statement);
+ return statement;
+ }
+
+ default String queryStoreCreateTableRawQuery() {
+ final String statement = "CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36) not null, `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint);";
+ log.trace("mapped create query store table statement: {}", statement);
+ return statement;
+ }
+
+ default String queryStoreCreateHashTableProcedureRawQuery() {
+ final String statement = "CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\\'\\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \\',\\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END;";
+ log.trace("mapped create query store hash_table procedure statement: {}", statement);
+ return statement;
+ }
+
+ default String queryStoreCreateStoreQueryProcedureRawQuery() {
+ final String statement = "CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;";
+ log.trace("mapped create query store store_query procedure statement: {}", statement);
+ return statement;
+ }
+
+ default String queryStoreCreateInternalStoreQueryProcedureRawQuery() {
+ final String statement = "CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;";
+ log.trace("mapped create query store _store_query procedure statement: {}", statement);
+ return statement;
+ }
+
+ default String queryStoreStoreQueryRawQuery() {
+ final String statement = "{call _store_query(?, ?, ?, ?)}";
+ log.trace("mapped store query statement: {}", statement);
+ return statement;
+ }
+
+ default String queryStoreUpdateQueryRawQuery() {
+ final String statement = "UPDATE `qs_queries` SET `is_persisted` = ? WHERE `id` = ?";
+ log.trace("mapped update query statement: {}", statement);
+ return statement;
+ }
+
+ default String queryStoreDeleteStaleQueriesRawQuery() {
+ final String statement = "DELETE FROM `qs_queries` WHERE `is_persisted` = false AND ABS(DATEDIFF(`created`, NOW())) >= 1";
+ log.trace("mapped delete stale queries statement: {}", statement);
+ return statement;
+ }
+
+ default String queryStoreFindQueryRawQuery() {
+ final String statement = "SELECT `id`, `created`, `created_by`, `query`, `query_hash`, `result_hash`, `result_number`, `is_persisted`, `executed` FROM `qs_queries` q WHERE q.`id` = ?";
+ log.trace("mapped find query statement: {}", statement);
+ return statement;
+ }
+
default String databaseTablesSelectRawQuery() {
final String statement = "SELECT DISTINCT t.`TABLE_NAME` FROM information_schema.TABLES t WHERE t.`TABLE_SCHEMA` = ? AND t.`TABLE_TYPE` = 'SYSTEM VERSIONED' AND t.`TABLE_NAME` != 'qs_queries' ORDER BY t.`TABLE_NAME` ASC";
log.trace("mapped select tables statement: {}", statement);
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java
index 5acca0018de6bc3e29de49aea9f1da4ae95ff492..f4bef4f067706688dc30234a3a375678214cdb8f 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java
@@ -20,12 +20,10 @@ public interface ViewService {
* @return The list of view metadata.
* @throws SQLException
* @throws DatabaseMalformedException
- * @throws ViewMalformedException
* @throws ViewNotFoundException
- * @throws ViewSchemaException
*/
List<ViewDto> getSchemas(PrivilegedDatabaseDto database) throws SQLException, DatabaseMalformedException,
- ViewMalformedException, ViewNotFoundException, ViewSchemaException;
+ ViewNotFoundException;
/**
* Creates a view in the given data database.
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java
index 8c52e020103db210744884e1f5973f67a8e44861..4fdb8c32a69eac23dc05b1982f1abbbc4893d985 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java
@@ -40,17 +40,25 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce
final Connection connection = dataSource.getConnection();
try {
/* create user if not exists */
+ long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseCreateUserQuery(user.getUsername(), user.getPassword()))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
/* grant access */
final String grants = access != AccessTypeDto.READ ? grantDefaultWrite : grantDefaultRead;
+ start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseGrantPrivilegesQuery(user.getUsername(), grants))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
/* grant query store */
- connection.prepareStatement("GRANT EXECUTE ON PROCEDURE `store_query` TO `" + user.getUsername() + "`@`%`;")
+ start = System.currentTimeMillis();
+ connection.prepareStatement(mariaDbMapper.databaseGrantProcedureQuery(user.getUsername(), "store_query"))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
/* apply access rights */
- connection.prepareStatement("FLUSH PRIVILEGES;");
+ start = System.currentTimeMillis();
+ connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery());
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -71,10 +79,12 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce
try {
/* grant access */
final String grants = access != AccessTypeDto.READ ? grantDefaultWrite : grantDefaultRead;
+ final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseGrantPrivilegesQuery(user.getUsername(), grants))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
/* apply access rights */
- connection.prepareStatement("FLUSH PRIVILEGES;");
+ connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery());
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -93,10 +103,15 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce
final Connection connection = dataSource.getConnection();
try {
/* revoke access */
- connection.prepareStatement("REVOKE ALL PRIVILEGES ON *.* FROM `" + user.getUsername() + "`@`%`;")
+ long start = System.currentTimeMillis();
+ connection.prepareStatement(mariaDbMapper.databaseRevokePrivilegesQuery(user.getUsername()))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
/* apply access rights */
- connection.prepareStatement("FLUSH PRIVILEGES;");
+ start = System.currentTimeMillis();
+ connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery())
+ .execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java
index 4ee483ad5e11b4bbc3499e0d98f255e666bec1ea..2d58744a21e9c06fa6fee111ba06240d87e44bd8 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java
@@ -43,8 +43,10 @@ public class DatabaseServiceMariaDbImpl extends HibernateConnector implements Da
final Connection connection = dataSource.getConnection();
try {
/* create database if not exists */
+ final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseCreateDatabaseQuery(data.getInternalName()))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -77,8 +79,10 @@ public class DatabaseServiceMariaDbImpl extends HibernateConnector implements Da
final Connection connection = dataSource.getConnection();
try {
/* update user password */
+ final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.databaseSetPasswordQuery(data.getUsername(), data.getPassword()))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java
index 6a583b274affc20d6871e7358e842d77a88f2c4c..22bb599c6090cfe75f152d7f0a2a8f26e03c71b4 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java
@@ -12,6 +12,7 @@ import org.springframework.stereotype.Service;
public abstract class HibernateConnector {
public static ComboPooledDataSource getPrivilegedDataSource(PrivilegedContainerDto container, String databaseName) {
+ final long start = System.currentTimeMillis();
final ComboPooledDataSource dataSource = new ComboPooledDataSource();
dataSource.setJdbcUrl(url(container, databaseName));
dataSource.setUser(container.getUsername());
@@ -21,7 +22,7 @@ public abstract class HibernateConnector {
dataSource.setAcquireIncrement(5);
dataSource.setMaxPoolSize(20);
dataSource.setMaxStatements(100);
- log.trace("created pooled data source {} (user={}, password=(hidden))", url(container, databaseName), container.getUsername());
+ log.trace("created pooled data source {} in {} ms (user={}, password=(hidden))", url(container, databaseName), System.currentTimeMillis() - start, container.getUsername());
return dataSource;
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
index f2675d4e5b537d751743ae70ea7c366e19a05351..0b1dc7caa1cd7f530d6d8590ffb09225e8048ff4 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
@@ -46,7 +46,9 @@ public class QueueServiceRabbitMqImpl extends HibernateConnector implements Queu
dataMapper.prepareStatementWithColumnTypeObject(preparedStatement, optional.get().getColumnType(), idx[0]++,
entry.getValue());
}
+ final long start = System.currentTimeMillis();
preparedStatement.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
log.trace("successfully inserted tuple");
} finally {
dataSource.close();
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java
index cc5840080b21ae8549632db7123e12ec72b1ee30..a0fbd1434c22be6a58d945d0fe669b5cc5be558f 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java
@@ -49,26 +49,32 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
final Connection connection = dataSource.getConnection();
try {
/* obtain only table metadata */
+ long start = System.currentTimeMillis();
final PreparedStatement statement1 = connection.prepareStatement(mariaDbMapper.databaseTableSelectRawQuery());
statement1.setString(1, database.getInternalName());
statement1.setString(2, tableName);
log.trace("1={}, 2={}", database.getInternalName(), tableName);
TableDto table = dataMapper.schemaResultSetToTable(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), statement1.executeQuery());
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
/* obtain columns metadata */
+ start = System.currentTimeMillis();
final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery());
statement2.setString(1, database.getInternalName());
statement2.setString(2, tableName);
log.trace("1={}, 2={}", database.getInternalName(), tableName);
final ResultSet resultSet2 = statement2.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
while (resultSet2.next()) {
table = dataMapper.resultSetToTable(resultSet2, table, queryConfig);
}
/* obtain check constraints metadata */
+ start = System.currentTimeMillis();
final PreparedStatement statement3 = connection.prepareStatement(mariaDbMapper.columnsCheckConstraintSelectRawQuery());
statement3.setString(1, database.getInternalName());
statement3.setString(2, tableName);
log.trace("1={}, 2={}", database.getInternalName(), tableName);
final ResultSet resultSet3 = statement3.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
while (resultSet3.next()) {
final String clause = resultSet3.getString(1);
table.getConstraints()
@@ -77,11 +83,13 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
log.trace("found check clause: {}", clause);
}
/* obtain column constraints metadata */
+ start = System.currentTimeMillis();
final PreparedStatement statement4 = connection.prepareStatement(mariaDbMapper.databaseTableConstraintsSelectRawQuery());
statement4.setString(1, database.getInternalName());
statement4.setString(2, tableName);
log.trace("1={}, 2={}", database.getInternalName(), tableName);
final ResultSet resultSet4 = statement4.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
while (resultSet4.next()) {
table = dataMapper.resultSetToConstraint(resultSet4, table);
for (UniqueDto uk : table.getConstraints().getUniques()) {
@@ -122,11 +130,13 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
final DatabaseDto database = metadataMapper.privilegedDatabaseDtoToDatabaseDto(privilegedDatabase);
try {
/* obtain only view metadata */
+ long start = System.currentTimeMillis();
final PreparedStatement statement1 = connection.prepareStatement(mariaDbMapper.databaseViewSelectRawQuery());
statement1.setString(1, database.getInternalName());
statement1.setString(2, viewName);
log.trace("1={}, 2={}", database.getInternalName(), viewName);
final ResultSet resultSet1 = statement1.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
if (!resultSet1.next()) {
throw new ViewNotFoundException("Failed to find view in the information schema");
}
@@ -136,11 +146,13 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
view.setCreator(database.getCreator());
view.setCreatedBy(database.getCreator().getId());
/* obtain view columns */
+ start = System.currentTimeMillis();
final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery());
statement2.setString(1, database.getInternalName());
statement2.setString(2, viewName);
log.trace("1={}, 2={}", database.getInternalName(), viewName);
final ResultSet resultSet2 = statement2.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
TableDto tmp = TableDto.builder()
.columns(new LinkedList<>())
.build();
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java
index 3d19276196895efc00834b60383ddfa226a46bc3..53d93d35dfaa11d4ec40ef5e00b73050ebea91c5 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java
@@ -68,16 +68,26 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final Connection connection = dataSource.getConnection();
try {
/* create query store */
- connection.prepareStatement("CREATE SEQUENCE `qs_queries_seq` NOCACHE;")
+ long start = System.currentTimeMillis();
+ connection.prepareStatement(mariaDbMapper.queryStoreCreateSequenceRawQuery())
.execute();
- connection.prepareStatement("CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36) not null, `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint );")
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ start = System.currentTimeMillis();
+ connection.prepareStatement(mariaDbMapper.queryStoreCreateTableRawQuery())
.execute();
- connection.prepareStatement("CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\\'\\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \\',\\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END;")
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ start = System.currentTimeMillis();
+ connection.prepareStatement(mariaDbMapper.queryStoreCreateHashTableProcedureRawQuery())
.execute();
- connection.prepareStatement("CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;")
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ start = System.currentTimeMillis();
+ connection.prepareStatement(mariaDbMapper.queryStoreCreateStoreQueryProcedureRawQuery())
.execute();
- connection.prepareStatement("CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;")
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ start = System.currentTimeMillis();
+ connection.prepareStatement(mariaDbMapper.queryStoreCreateInternalStoreQueryProcedureRawQuery())
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -130,12 +140,14 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database);
final Connection connection = dataSource.getConnection();
try {
+ final long start = System.currentTimeMillis();
final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.filterToGetQueriesRawQuery(filterPersisted));
if (filterPersisted != null) {
statement.setBoolean(1, filterPersisted);
log.trace("filter persisted only {}", filterPersisted);
}
final ResultSet resultSet = statement.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
final List<QueryDto> queries = new LinkedList<>();
while (resultSet.next()) {
final QueryDto query = dataMapper.resultSetToQueryDto(resultSet);
@@ -168,12 +180,18 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final Connection connection = dataSource.getConnection();
try {
/* export to data database sidecar */
+ long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.subsetToRawTemporaryViewQuery(viewName, query.getQuery()))
.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.subsetToRawExportQuery(viewName, timestamp, filePath))
.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.dropViewRawQuery(viewName))
.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -191,8 +209,10 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database);
final Connection connection = dataSource.getConnection();
try {
+ final long start = System.currentTimeMillis();
final PreparedStatement preparedStatement = connection.prepareStatement(statement);
final ResultSet resultSet = preparedStatement.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
return dataMapper.resultListToQueryResultDto(columns, resultSet);
} catch (SQLException e) {
log.error("Failed to execute and map time-versioned query: {}", e.getMessage());
@@ -208,8 +228,10 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database);
final Connection connection = dataSource.getConnection();
try {
+ final long start = System.currentTimeMillis();
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.countRawSelectQuery(statement, timestamp))
.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
return mariaDbMapper.resultSetToNumber(resultSet);
} catch (SQLException e) {
log.error("Failed to map object: {}", e.getMessage());
@@ -225,9 +247,11 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database);
final Connection connection = dataSource.getConnection();
try {
- final PreparedStatement preparedStatement = connection.prepareStatement("SELECT `id`, `created`, `created_by`, `query`, `query_hash`, `result_hash`, `result_number`, `is_persisted`, `executed` FROM `qs_queries` q WHERE q.`id` = ?");
+ final long start = System.currentTimeMillis();
+ final PreparedStatement preparedStatement = connection.prepareStatement(mariaDbMapper.queryStoreFindQueryRawQuery());
preparedStatement.setLong(1, queryId);
final ResultSet resultSet = preparedStatement.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
if (!resultSet.next()) {
throw new QueryNotFoundException("Failed to find query");
}
@@ -255,12 +279,14 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final Connection connection = dataSource.getConnection();
try {
/* insert query into query store */
- final CallableStatement callableStatement = connection.prepareCall("{call _store_query(?, ?, ?, ?)}");
+ final long start = System.currentTimeMillis();
+ final CallableStatement callableStatement = connection.prepareCall(mariaDbMapper.queryStoreStoreQueryRawQuery());
callableStatement.setString(1, String.valueOf(userId));
callableStatement.setString(2, query);
callableStatement.setTimestamp(3, Timestamp.from(timestamp));
callableStatement.registerOutParameter(4, Types.BIGINT);
callableStatement.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
queryId = callableStatement.getLong(4);
callableStatement.close();
log.info("Stored query with id {} in database with name {}", queryId, database.getInternalName());
@@ -282,10 +308,12 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final Connection connection = dataSource.getConnection();
try {
/* update query */
- final PreparedStatement preparedStatement = connection.prepareStatement("UPDATE `qs_queries` SET `is_persisted` = ? WHERE `id` = ?");
+ final long start = System.currentTimeMillis();
+ final PreparedStatement preparedStatement = connection.prepareStatement(mariaDbMapper.queryStoreUpdateQueryRawQuery());
preparedStatement.setBoolean(1, persist);
preparedStatement.setLong(2, queryId);
preparedStatement.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
} catch (SQLException e) {
log.error("Failed to (un-)persist query: {}", e.getMessage());
throw new QueryStorePersistException("Failed to (un-)persist query", e);
@@ -300,8 +328,10 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database);
final Connection connection = dataSource.getConnection();
try {
- connection.prepareStatement("DELETE FROM `qs_queries` WHERE `is_persisted` = false AND ABS(DATEDIFF(`created`, NOW())) >= 1")
+ final long start = System.currentTimeMillis();
+ connection.prepareStatement(mariaDbMapper.queryStoreDeleteStaleQueriesRawQuery())
.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
} catch (SQLException e) {
log.error("Failed to delete stale queries: {}", e.getMessage());
throw new QueryStoreGCException("Failed to delete stale queries: " + e.getMessage(), e);
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
index 4dacc1e094f0c0f11e6d9949ee4cbf2fbc9e8621..97778e9e11679bc349ef6037fe8c7a9150cf530a 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
@@ -25,6 +25,7 @@ import org.apache.commons.lang3.RandomStringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import java.io.File;
import java.sql.*;
import java.time.Instant;
import java.util.*;
@@ -60,9 +61,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final List<TableDto> tables = new LinkedList<>();
try {
/* inspect tables before views */
+ final long start = System.currentTimeMillis();
final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.databaseTablesSelectRawQuery());
statement.setString(1, database.getInternalName());
final ResultSet resultSet1 = statement.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
while (resultSet1.next()) {
final String tableName = resultSet1.getString(1);
if (database.getTables().stream().anyMatch(t -> t.getInternalName().equals(tableName))) {
@@ -92,8 +95,10 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final TableStatisticDto statistic;
try {
/* obtain statistic */
+ final long start = System.currentTimeMillis();
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.tableColumnStatisticsSelectRawQuery(table.getColumns(), table.getInternalName()))
.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
statistic = dataMapper.resultSetToTableStatistic(resultSet);
final TableDto tmpTable = schemaService.inspectTable(table.getDatabase(), table.getInternalName());
statistic.setAvgRowLength(tmpTable.getAvgRowLength());
@@ -129,8 +134,10 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final Connection connection = dataSource.getConnection();
try {
/* create table if not exists */
+ final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.tableCreateDtoToCreateTableRawQuery(data))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -156,8 +163,10 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final Connection connection = dataSource.getConnection();
try {
/* create table if not exists */
+ final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.dropTableRawQuery(tableName))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -177,12 +186,16 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final QueryResultDto queryResult;
try {
/* find table data */
+ long start = System.currentTimeMillis();
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectDatasetRawQuery(
table.getDatabase().getInternalName(), table.getInternalName(), table.getColumns(),
timestamp, size, page))
.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
+ start = System.currentTimeMillis();
connection.commit();
queryResult = dataMapper.resultListToQueryResultDto(table.getColumns(), resultSet);
+ log.debug("mapped result in {} ms", System.currentTimeMillis() - start);
} catch (SQLException e) {
connection.rollback();
log.error("Failed to find data from table {}.{}: {}", table.getDatabase().getInternalName(), table.getInternalName(), e.getMessage());
@@ -203,9 +216,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final List<TableHistoryDto> history;
try {
/* find table data */
+ final long start = System.currentTimeMillis();
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectHistoryRawQuery(
table.getDatabase().getInternalName(), table.getInternalName(), size))
.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
history = dataMapper.resultSetToTableHistory(resultSet);
connection.commit();
} catch (SQLException e) {
@@ -227,9 +242,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final Long queryResult;
try {
/* find table data */
+ final long start = System.currentTimeMillis();
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectCountRawQuery(
table.getDatabase().getInternalName(), table.getInternalName(), timestamp))
.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
queryResult = mariaDbMapper.resultSetToNumber(resultSet);
connection.commit();
} catch (SQLException e) {
@@ -253,9 +270,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final Connection connection = dataSource.getConnection();
try {
/* import tuple */
- data.setLocation(s3Config.getS3FilePath() + "/" + data.getLocation());
+ data.setLocation(s3Config.getS3FilePath() + File.separator + data.getLocation());
+ final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.datasetToRawInsertQuery(table.getDatabase().getInternalName(), table, data))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -283,7 +302,9 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
getColumnType(table.getColumns(), column), idx[0], column, data.getKeys().get(column));
idx[0]++;
}
+ final long start = System.currentTimeMillis();
statement.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -325,7 +346,9 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
getColumnType(table.getColumns(), entry.getKey()), idx[0], entry.getKey(), entry.getValue());
idx[0]++;
}
+ final long start = System.currentTimeMillis();
statement.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -359,7 +382,9 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
getColumnType(table.getColumns(), entry.getKey()), idx[0], entry.getKey(), entry.getValue());
idx[0]++;
}
+ final long start = System.currentTimeMillis();
statement.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -392,9 +417,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final Connection connection = dataSource.getConnection();
try {
/* export to data database sidecar */
+ final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.tableOrViewToRawExportQuery(table.getDatabase().getInternalName(),
table.getInternalName(), table.getColumns(), timestamp, filePath))
.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java
index 3cdba35f085491334300cc3fb5a3ab3ee30801fd..06cf42ae6ed0b49f56f63648cd3286b4226f727a 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang3.RandomStringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import java.io.File;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -63,7 +64,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
@Override
public List<ViewDto> getSchemas(PrivilegedDatabaseDto database) throws SQLException, DatabaseMalformedException,
- ViewMalformedException, ViewNotFoundException, ViewSchemaException {
+ ViewNotFoundException {
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database);
final Connection connection = dataSource.getConnection();
final List<ViewDto> views = new LinkedList<>();
@@ -71,7 +72,9 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
/* inspect tables before views */
final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.databaseViewsSelectRawQuery());
statement.setString(1, database.getInternalName());
+ final long start = System.currentTimeMillis();
final ResultSet resultSet1 = statement.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
while (resultSet1.next()) {
final String viewName = resultSet1.getString(1);
if (database.getViews().stream().anyMatch(v -> v.getInternalName().equals(viewName))) {
@@ -117,8 +120,10 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
.build();
try {
/* create view if not exists */
+ final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.viewCreateRawQuery(view.getInternalName(), data.getQuery()))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
/* select view columns */
final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery());
statement2.setString(1, database.getInternalName());
@@ -151,10 +156,12 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
.stream()
.map(metadataMapper::viewColumnDtoToColumnDto)
.toList();
+ final long start = System.currentTimeMillis();
final ResultSet resultSet = connection.prepareStatement(
mariaDbMapper.selectDatasetRawQuery(view.getDatabase().getInternalName(),
view.getInternalName(), mappedColumns, timestamp, size, page))
.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
queryResult = dataMapper.resultListToQueryResultDto(mappedColumns, resultSet);
queryResult.setId(view.getId());
connection.commit();
@@ -174,8 +181,10 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
final Connection connection = dataSource.getConnection();
try {
/* drop view if exists */
+ final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.dropViewRawQuery(view.getInternalName()))
.execute();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
@@ -196,9 +205,11 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
final Long queryResult;
try {
/* find view data */
+ final long start = System.currentTimeMillis();
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectCountRawQuery(
view.getDatabase().getInternalName(), view.getInternalName(), timestamp))
.executeQuery();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
queryResult = mariaDbMapper.resultSetToNumber(resultSet);
connection.commit();
} catch (SQLException e) {
@@ -217,7 +228,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
throws SQLException, QueryMalformedException, StorageNotFoundException, StorageUnavailableException,
RemoteUnavailableException, SidecarExportException {
final String fileName = RandomStringUtils.randomAlphabetic(40) + ".csv";
- final String filePath = s3Config.getS3FilePath() + "/" + fileName;
+ final String filePath = s3Config.getS3FilePath() + File.separator + fileName;
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database);
final Connection connection = dataSource.getConnection();
try {
@@ -226,9 +237,11 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
.stream()
.map(metadataMapper::viewColumnDtoToColumnDto)
.toList();
+ final long start = System.currentTimeMillis();
connection.prepareStatement(mariaDbMapper.tableOrViewToRawExportQuery(database.getInternalName(),
view.getInternalName(), columns, timestamp, filePath))
.executeUpdate();
+ log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
connection.commit();
} catch (SQLException e) {
connection.rollback();
diff --git a/dbrepo-gateway-service/dbrepo.conf b/dbrepo-gateway-service/dbrepo.conf
index f9c0001cebfbba5bbb842984377fb01197617d96..de8eaa417bbe422999436e83dc24f8fb66f9b1d5 100644
--- a/dbrepo-gateway-service/dbrepo.conf
+++ b/dbrepo-gateway-service/dbrepo.conf
@@ -134,7 +134,7 @@ server {
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://data;
- proxy_read_timeout 90;
+ proxy_read_timeout 600;
}
location ~ /api/(database|concept|container|identifier|image|message|license|oai|ontology|unit|user) {
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java
index 307c166febb3634a08991c7b5d571bcf7bf072e1..da1c28918576b0a0bf606b23c15e5d0a02a63820 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java
@@ -38,9 +38,9 @@ public class BrokerServiceGatewayImpl implements BrokerServiceGateway {
response = restTemplate.exchange(path, HttpMethod.PUT, new HttpEntity<>(data), Void.class);
} catch (HttpServerErrorException e) {
log.error("Failed to grant topic permissions: {}", e.getMessage());
- throw new BrokerServiceConnectionException("Failed to grant topic permissions: " + e.getMessage());
+ throw new BrokerServiceConnectionException("Failed to grant topic permissions: " + e.getMessage(), e);
} catch (Exception e) {
- log.error("Failed to grant topic permissions: unexpected response: {}", e.getMessage());
+ log.error("Failed to grant topic permissions: unexpected response: {}", e.getMessage(), e);
throw new BrokerServiceException("Failed to grant topic permissions: unexpected response: " + e.getMessage(), e);
}
if (!response.getStatusCode().equals(HttpStatus.CREATED) && !response.getStatusCode().equals(HttpStatus.NO_CONTENT)) {
@@ -59,7 +59,7 @@ public class BrokerServiceGatewayImpl implements BrokerServiceGateway {
response = restTemplate.exchange(path, HttpMethod.PUT, new HttpEntity<>(data), Void.class);
} catch (HttpServerErrorException e) {
log.error("Failed to grant virtual host permissions: {}", e.getMessage());
- throw new BrokerServiceConnectionException("Failed to grant virtual host permissions: " + e.getMessage());
+ throw new BrokerServiceConnectionException("Failed to grant virtual host permissions: " + e.getMessage(), e);
} catch (Exception e) {
log.error("Failed to grant virtual host permissions: unexpected response: {}", e.getMessage());
throw new BrokerServiceException("Failed to grant virtual host permissions: unexpected response: " + e.getMessage(), e);
@@ -80,7 +80,7 @@ public class BrokerServiceGatewayImpl implements BrokerServiceGateway {
response = restTemplate.exchange(path, HttpMethod.PUT, new HttpEntity<>(data), Void.class);
} catch (HttpServerErrorException e) {
log.error("Failed to grant exchange permissions: {}", e.getMessage());
- throw new BrokerServiceConnectionException("Failed to grant exchange permissions: " + e.getMessage());
+ throw new BrokerServiceConnectionException("Failed to grant exchange permissions: " + e.getMessage(), e);
} catch (Exception e) {
log.error("Failed to grant exchange permissions: unexpected response: {}", e.getMessage());
throw new BrokerServiceException("Failed to grant exchange permissions: unexpected response: " + e.getMessage(), e);
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java
index 542d9c981d82f6b8a300808ed94ca40034c95851..8fd957f330a947793f9bfc4e4fcbb08616bbf90f 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java
@@ -37,7 +37,7 @@ public class CrossrefGatewayImpl implements CrossrefGateway {
response = restTemplate.exchange(gatewayConfig.getCrossRefEndpoint() + path, HttpMethod.GET, HttpEntity.EMPTY, CrossrefDto.class);
} catch (HttpServerErrorException e) {
log.error("Failed to retrieve crossref metadata: {}", e.getMessage());
- throw new DoiNotFoundException("Failed to retrieve crossref metadata: " + e.getMessage());
+ throw new DoiNotFoundException("Failed to retrieve crossref metadata: " + e.getMessage(), e);
}
return response.getBody();
}
diff --git a/dbrepo-ui/components/database/DatabaseCard.vue b/dbrepo-ui/components/database/DatabaseCard.vue
index 9485fdee101584250d10bc1b8b959565a6699541..48aefa7493ce5beb52d8ec62cf794d64368b8489 100644
--- a/dbrepo-ui/components/database/DatabaseCard.vue
+++ b/dbrepo-ui/components/database/DatabaseCard.vue
@@ -4,11 +4,20 @@
:to="`/database/${database.id}/info`"
variant="flat"
rounded="0"
- :href="`/database/${database.id}`">
+ :href="`/database/${database.id}`"
+ @click="loading = true">
<v-divider class="mx-4" />
- <v-card-title
- class="text-primary text-decoration-underline"
- v-text="formatTitle(database)" />
+ <v-card-title>
+ <span
+ class="text-primary text-decoration-underline"
+ v-text="formatTitle(database)" />
+ <v-progress-circular
+ v-if="loading"
+ color="primary"
+ size="24"
+ class="ml-1"
+ indeterminate />
+ </v-card-title>
<v-card-subtitle
v-text="formatCreators(database)" />
<v-card-text>
@@ -68,6 +77,11 @@
import { formatLanguage } from '@/utils'
export default {
+ data() {
+ return {
+ loading: false
+ }
+ },
props: {
database: {
default: () => {
diff --git a/dbrepo-ui/components/database/DatabaseToolbar.vue b/dbrepo-ui/components/database/DatabaseToolbar.vue
index 5a86bbaa01410fb068533930af6636fc9f20d1d7..0c87e5599c688b3c4feebfbadb09a8868071d54d 100644
--- a/dbrepo-ui/components/database/DatabaseToolbar.vue
+++ b/dbrepo-ui/components/database/DatabaseToolbar.vue
@@ -10,21 +10,21 @@
<span
v-if="database && $vuetify.display.lgAndUp"
v-text="database.name" />
- <v-tooltip
- v-if="database"
- bottom>
- <template v-slot:activator="{ props }">
- <v-icon
- class="mr-2"
- size="small"
- right
- :color="database.is_public ? 'success' : 'chip'"
- v-bind="props">
- {{ database.is_public ? 'mdi-lock-open-outline' : 'mdi-lock-outline' }}
- </v-icon>
- </template>
- <span>{{ $t('toolbars.database.' + (database.is_public ? 'public' : 'private')) }}</span>
- </v-tooltip>
+ <v-chip
+ v-if="database && database.is_public"
+ size="small"
+ class="ml-2"
+ color="success"
+ :text="$t('toolbars.database.public')"
+ variant="outlined" />
+ <v-chip
+ v-if="database && !database.is_public"
+ size="small"
+ class="ml-2"
+ :color="colorVariant"
+ variant="outlined"
+ :text="$t('toolbars.database.private')"
+ flat />
</v-toolbar-title>
<v-spacer />
<v-btn
@@ -33,6 +33,14 @@
color="tertiary"
:variant="buttonVariant"
:text="$t('toolbars.database.dashboard.permanent') + ($vuetify.display.lgAndUp ? ' ' + $t('toolbars.database.dashboard.xl') : '')" />
+ <v-btn
+ v-if="canCreateTable"
+ :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-table-large-plus' : null"
+ color="secondary"
+ variant="flat"
+ :text="($vuetify.display.lgAndUp ? $t('toolbars.database.create-table.xl') + ' ' : '') + $t('toolbars.database.create-table.permanent')"
+ class="mr-2"
+ :to="`/database/${$route.params.database_id}/table/create/dataset`" />
<v-btn
v-if="canCreateSubset"
:prepend-icon="$vuetify.display.lgAndUp ? 'mdi-wrench' : null"
@@ -49,14 +57,6 @@
:text="($vuetify.display.lgAndUp ? $t('toolbars.database.create-view.xl') + ' ' : '') + $t('toolbars.database.create-view.permanent')"
class="mr-2 white--text"
:to="`/database/${$route.params.database_id}/view/create`" />
- <v-btn
- v-if="canCreateTable"
- :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-table-large-plus' : null"
- color="secondary"
- variant="flat"
- :text="($vuetify.display.lgAndUp ? $t('toolbars.database.create-table.xl') + ' ' : '') + $t('toolbars.database.create-table.permanent')"
- class="mr-2"
- :to="`/database/${$route.params.database_id}/table/create/dataset`" />
<v-btn
v-if="canCreateIdentifier"
:prepend-icon="$vuetify.display.lgAndUp ? 'mdi-identifier' : null"
@@ -117,6 +117,9 @@ export default {
roles () {
return this.userStore.getRoles
},
+ colorVariant () {
+ return this.isContrastTheme ? '' : (this.isDarkTheme ? 'tertiary' : 'secondary')
+ },
canCreateIdentifier () {
if (!this.roles) {
return false
diff --git a/dbrepo-ui/components/identifier/Citation.vue b/dbrepo-ui/components/identifier/Citation.vue
index 8cd96902d48009da22cd761849be1962ba07c248..6f35ac915e255610965015ccd16adcf1dad03597 100644
--- a/dbrepo-ui/components/identifier/Citation.vue
+++ b/dbrepo-ui/components/identifier/Citation.vue
@@ -10,8 +10,8 @@
<v-select
v-model="style"
:items="styles"
- item-title="style"
- item-value="accept"
+ item-title="title"
+ item-value="value"
dense
variant="outlined"
single-line />
@@ -33,39 +33,38 @@ export default {
return {
loading: false,
styles: [
- { style: 'APA', accept: 'text/bibliography;style=apa' },
- { style: 'IEEE', accept: 'text/bibliography;style=ieee' },
- { style: 'BibTeX', accept: 'text/bibliography;style=bibtex' }
+ { title: 'APA', value: 'text/bibliography;style=apa' },
+ { title: 'IEEE', value: 'text/bibliography;style=ieee' },
+ { title: 'BibTeX', value: 'text/bibliography;style=bibtex' }
],
- style: null,
+ style: 'text/bibliography;style=apa',
citation: null
}
},
watch: {
style () {
- this.loadCitation(this.style)
+ this.loadCitation()
},
pid () {
- this.loadCitation(this.style)
+ this.loadCitation()
}
},
mounted () {
- this.style = this.styles[0].accept
- this.loadCitation(null)
+ this.loadCitation()
},
methods: {
- loadCitation (accept) {
- if (!this.identifier || !accept) {
+ loadCitation () {
+ if (!this.identifier || !this.style) {
return
}
this.loading = true
const identifierService = useIdentifierService()
- identifierService.findOne(this.identifier.id, accept)
+ identifierService.findOne(this.identifier.id, this.style)
.then((citation) => {
this.citation = citation
this.loading = false
})
- .error(({code, message}) => {
+ .catch(({code, message}) => {
const toast = useToastInstance()
toast.error(this.$t(`${code}: ${message}`))
this.loading = false
diff --git a/dbrepo-ui/components/subset/SubsetList.vue b/dbrepo-ui/components/subset/SubsetList.vue
index 21d4e7263eedd942f8abfec1470dd3a13af9d53e..b977daffa5aaf71dbe2980b98761244e8a3627b0 100644
--- a/dbrepo-ui/components/subset/SubsetList.vue
+++ b/dbrepo-ui/components/subset/SubsetList.vue
@@ -1,7 +1,7 @@
<template>
<div>
<v-card
- v-if="!loadingSubsets && queries.length === 0"
+ v-if="!loadingSubsets && subsets.length === 0"
variant="flat"
rounded="0"
:text="$t('pages.database.subpages.subsets.empty')" />
@@ -14,7 +14,7 @@
<Loading />
</v-list-item>
<div
- v-for="(item, i) in queries"
+ v-for="(item, i) in subsets"
:key="`q-${i}`">
<v-divider v-if="i !== 0" class="mx-4" />
<v-list>
@@ -54,9 +54,7 @@ export default {
return {
loadingSubsets: false,
loadingIdentifiers: false,
- queries: [],
- identifiers: [],
- isAuthorizationError: false,
+ subsets: [],
cacheStore: useCacheStore(),
userStore: useUserStore()
}
@@ -77,8 +75,9 @@ export default {
this.loadingSubsets = true
const queryService = useQueryService()
queryService.findAll(this.$route.params.database_id, true)
- .then((queries) => {
- this.queries = queries
+ .then((subsets) => {
+ this.loadingSubsets = false
+ this.subsets = subsets
})
.catch(({code}) => {
this.loadingSubsets = false
@@ -88,9 +87,6 @@ export default {
}
toast.error(this.$t(code))
})
- .finally(() => {
- this.loadingSubsets = false
- })
},
title (query) {
if (query.identifiers.length === 0) {
diff --git a/dbrepo-ui/components/subset/SubsetToolbar.vue b/dbrepo-ui/components/subset/SubsetToolbar.vue
index b51cc1d089085b657cb72d689fbf9ef2c6776b36..5c5081a2f8f5c3944928bbd59cd76dddc2c79d89 100644
--- a/dbrepo-ui/components/subset/SubsetToolbar.vue
+++ b/dbrepo-ui/components/subset/SubsetToolbar.vue
@@ -1,6 +1,7 @@
<template>
<div>
- <v-toolbar flat>
+ <v-toolbar
+ flat>
<v-btn
class="mr-2"
variant="plain"
@@ -27,16 +28,16 @@
color="secondary"
variant="flat"
class="mb-1 ml-2"
- :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-content-save-outline' : null"
+ :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-star' : null"
:text="$t('toolbars.subset.save.permanent')"
@click.stop="save" />
<v-btn
v-if="canForgetQuery"
:loading="loadingSave"
- color="error"
+ color="warning"
variant="flat"
class="mb-1 ml-2"
- :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-trash-can-outline' : null"
+ :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-star-off' : null"
:text="$t('toolbars.subset.unsave.permanent')"
@click.stop="forget" />
<v-btn
@@ -116,7 +117,10 @@ export default {
if (!this.database) {
return false
}
- return this.database.is_public
+ if (this.database.is_public) {
+ return true
+ }
+ return this.access
},
identifier () {
/* mount pid */
@@ -154,7 +158,7 @@ export default {
return formatTimestampUTCLabel(this.subset.created)
},
result_visibility () {
- if (!this.database) {
+ if (!this.database || !this.subset) {
return false
}
if (this.database.is_public) {
diff --git a/dbrepo-ui/components/view/ViewToolbar.vue b/dbrepo-ui/components/view/ViewToolbar.vue
index c107c3c0ef006c2d5456f6db187908e092664c5a..c43b730397a977559f85b55f92e62b486b15aa6f 100644
--- a/dbrepo-ui/components/view/ViewToolbar.vue
+++ b/dbrepo-ui/components/view/ViewToolbar.vue
@@ -11,16 +11,17 @@
<v-spacer />
<v-btn
v-if="canDeleteView"
- prepend-icon="mdi-delete"
class="mr-2"
variant="flat"
- color="error"
- :text="$vuetify.display.lgAndUp ? $t('navigation.delete') : ''"
+ :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-delete' : null"
:loading="loadingDelete"
+ color="error"
+ :text="$t('navigation.delete')"
@click="deleteView" />
<v-btn
v-if="canCreatePid"
- prepend-icon="mdi-content-save-outline"
+ class="mr-2"
+ :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-content-save-outline' : null"
variant="flat"
color="primary"
:text="($vuetify.display.lgAndUp ? $t('toolbars.view.pid.xl') + ' ' : '') + $t('toolbars.view.pid.permanent')"
@@ -63,6 +64,10 @@ export default {
database () {
return this.cacheStore.getDatabase
},
+ buttonVariant () {
+ const runtimeConfig = useRuntimeConfig()
+ return this.$vuetify.theme.global.name.toLowerCase().endsWith('contrast') ? runtimeConfig.public.variant.button.contrast : runtimeConfig.public.variant.button.normal
+ },
view () {
if (!this.database) {
return null
diff --git a/dbrepo-ui/composables/axios-instance.ts b/dbrepo-ui/composables/axios-instance.ts
index a2b6ca4f6ef5b138473b7812b460981a6f7c7f2f..d4f274b717e938c3305bc20463c93d9e9474e079 100644
--- a/dbrepo-ui/composables/axios-instance.ts
+++ b/dbrepo-ui/composables/axios-instance.ts
@@ -8,7 +8,7 @@ export const useAxiosInstance = () => {
const userStore = useUserStore()
if (!instance) {
instance = axios.create({
- timeout: 10_000,
+ timeout: 90_000,
params: {},
headers: {
Accept: 'application/json',
diff --git a/dbrepo-ui/composables/identifier-service.ts b/dbrepo-ui/composables/identifier-service.ts
index a85f05a45c5d4211ca9374b874ccc367d5b56372..96e5610c8c11161b0a4c1687e630204f80e53554 100644
--- a/dbrepo-ui/composables/identifier-service.ts
+++ b/dbrepo-ui/composables/identifier-service.ts
@@ -2,7 +2,7 @@ import type {AxiosError, AxiosRequestConfig} from 'axios'
import {axiosErrorToApiError} from '@/utils'
export const useIdentifierService = (): any => {
- async function findOne(id: number, accept: string | null): Promise<IdentifierDto> {
+ async function findOne(id: number, accept: string): Promise<IdentifierDto> {
const axios = useAxiosInstance()
console.debug('find identifier with id', id)
const config: AxiosRequestConfig = {
diff --git a/dbrepo-ui/composables/query-service.ts b/dbrepo-ui/composables/query-service.ts
index fa18a55ec053bad22339a14bcb053b9a405f3716..abcd928b66d0350fd855ae380b959a33818b8f3d 100644
--- a/dbrepo-ui/composables/query-service.ts
+++ b/dbrepo-ui/composables/query-service.ts
@@ -13,6 +13,10 @@ export const useQueryService = (): any => {
resolve(response.data)
})
.catch((error) => {
+ if (error.response.status === 403) {
+ /* ignore */
+ resolve([])
+ }
console.error('Failed to find queries', error)
reject(axiosErrorToApiError(error))
})
@@ -77,7 +81,7 @@ export const useQueryService = (): any => {
const axios = useAxiosInstance()
console.debug('execute query in database with id', databaseId)
return new Promise<QueryResultDto>((resolve, reject) => {
- axios.post<QueryResultDto>(`/api/database/${databaseId}/subset`, data, {params: mapFilter(timestamp, page, size)})
+ axios.post<QueryResultDto>(`/api/database/${databaseId}/subset`, data, {params: mapFilter(timestamp, page, size), timeout: 600_000})
.then((response) => {
console.info('Executed query with id', response.data.id, ' in database with id', databaseId)
resolve(response.data)
@@ -93,7 +97,7 @@ export const useQueryService = (): any => {
const axios = useAxiosInstance()
console.debug('re-execute query in database with id', databaseId)
return new Promise<QueryResultDto>((resolve, reject) => {
- axios.get<QueryResultDto>(`/api/database/${databaseId}/subset/${queryId}/data`, { params: mapFilter(null, page, size), timeout: 30_000 })
+ axios.get<QueryResultDto>(`/api/database/${databaseId}/subset/${queryId}/data`, { params: mapFilter(null, page, size) })
.then((response) => {
console.info('Re-executed query in database with id', databaseId)
resolve(response.data)
@@ -109,7 +113,7 @@ export const useQueryService = (): any => {
const axios = useAxiosInstance()
console.debug('re-execute query in database with id', databaseId)
return new Promise<number>((resolve, reject) => {
- axios.head<void>(`/api/database/${databaseId}/subset/${queryId}/data`, { timeout: 30_000 })
+ axios.head<void>(`/api/database/${databaseId}/subset/${queryId}/data`)
.then((response) => {
const count: number = Number(response.headers['x-count'])
console.info('Found', count, 'tuples for query', queryId, 'in database with id', databaseId)
diff --git a/dbrepo-ui/composables/table-service.ts b/dbrepo-ui/composables/table-service.ts
index 88ab27b39ee93d632d4492ed4a2d2d776cc08e8c..ffd7ebcd6074cc23d9bfa9412e19e48ef87f8f37 100644
--- a/dbrepo-ui/composables/table-service.ts
+++ b/dbrepo-ui/composables/table-service.ts
@@ -71,7 +71,7 @@ export const useTableService = (): any => {
const axios = useAxiosInstance()
console.debug('get data for table with id', tableId, 'in database with id', databaseId);
return new Promise<QueryResultDto>((resolve, reject) => {
- axios.get<QueryResultDto>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, page, size), timeout: 30_000 })
+ axios.get<QueryResultDto>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, page, size) })
.then((response) => {
console.info('Got data for table with id', tableId, 'in database with id', databaseId)
resolve(response.data)
@@ -87,7 +87,7 @@ export const useTableService = (): any => {
const axios = useAxiosInstance()
console.debug('get data count for table with id', tableId, 'in database with id', databaseId);
return new Promise<number>((resolve, reject) => {
- axios.head<void>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, null, null), timeout: 30_000 })
+ axios.head<void>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, null, null) })
.then((response: AxiosResponse<void>) => {
const count: number = Number(response.headers['x-count'])
console.info('Found' + count + 'in table with id', tableId, 'in database with id', databaseId)
@@ -191,7 +191,7 @@ export const useTableService = (): any => {
const axios = useAxiosInstance()
console.debug('suggest semantic entities for table column with id', columnId, 'of table with id', tableId, 'of database with id', databaseId)
return new Promise<TableColumnEntityDto[]>((resolve, reject) => {
- axios.get<TableColumnEntityDto[]>(`/api/database/${databaseId}/table/${tableId}/column/${columnId}/suggest`, {timeout: 10000})
+ axios.get<TableColumnEntityDto[]>(`/api/database/${databaseId}/table/${tableId}/column/${columnId}/suggest`)
.then((response) => {
console.info('Suggested semantic entities for table column with id', columnId, 'of table with id', tableId, 'of database with id', databaseId)
resolve(response.data)
diff --git a/dbrepo-ui/composables/view-service.ts b/dbrepo-ui/composables/view-service.ts
index adc76e32a1407f7e893cbe92692ec0b2f597591b..642a7c6e51ca7344dae72dd3ee12550c84c893a3 100644
--- a/dbrepo-ui/composables/view-service.ts
+++ b/dbrepo-ui/composables/view-service.ts
@@ -37,7 +37,7 @@ export const useViewService = (): any => {
const axios = useAxiosInstance()
console.debug('re-execute view with id', viewId, 'in database with id', databaseId)
return new Promise<QueryResultDto>((resolve, reject) => {
- axios.get<QueryResultDto>(`/api/database/${databaseId}/view/${viewId}/data`, { params: {page, size}, timeout: 30_000 })
+ axios.get<QueryResultDto>(`/api/database/${databaseId}/view/${viewId}/data`, { params: {page, size} })
.then((response) => {
console.info('Re-executed view with id', viewId, 'in database with id', databaseId)
resolve(response.data)
@@ -53,7 +53,7 @@ export const useViewService = (): any => {
const axios = useAxiosInstance()
console.debug('re-execute view with id', viewId, 'in database with id', databaseId)
return new Promise<number>((resolve, reject) => {
- axios.head<number>(`/api/database/${databaseId}/view/${viewId}/data`, { timeout: 30_000 })
+ axios.head<number>(`/api/database/${databaseId}/view/${viewId}/data`)
.then((response) => {
const count: number = Number(response.headers['x-count'])
console.info('Found', count, 'tuples for view with id', viewId, 'in database with id', databaseId)
diff --git a/dbrepo-ui/locales/en-US.json b/dbrepo-ui/locales/en-US.json
index cf8dcfd03e8df91fd4ba737c4c3662dec41402f0..be8ddfcc2317d441d3a8b6edacd4b3f5a663c6af 100644
--- a/dbrepo-ui/locales/en-US.json
+++ b/dbrepo-ui/locales/en-US.json
@@ -1382,7 +1382,7 @@
},
"subset": {
"save": {
- "permanent": "Save"
+ "permanent": "Star"
},
"export": {
"data": {
@@ -1399,7 +1399,7 @@
"permanent": "PID"
},
"unsave": {
- "permanent": "Unsave"
+ "permanent": "Unstar"
}
},
"view": {
diff --git a/dbrepo-ui/pages/database/[database_id]/settings.vue b/dbrepo-ui/pages/database/[database_id]/settings.vue
index 61e9dc62449b48cba17d77b4bb717e8164c9f257..a79ad48c4193c9b4f0fbc83f84b3a2c0db3a6193 100644
--- a/dbrepo-ui/pages/database/[database_id]/settings.vue
+++ b/dbrepo-ui/pages/database/[database_id]/settings.vue
@@ -422,7 +422,7 @@ export default {
databaseService.updateVisibility(this.$route.params.database_id, this.modifyVisibility)
.then((database) => {
const toast = useToastInstance()
- toast.success('success.database.visibility')
+ toast.success(this.$t('success.database.visibility'))
this.cacheStore.setDatabase(database)
})
.catch(() => {
diff --git a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue
index f740416faaf70387ed73935754c777435f58fa67..4902e2c54f243c7a2e4f3df90f3aa16cf7c26753 100644
--- a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue
+++ b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue
@@ -14,6 +14,15 @@
v-else
v-text="executionUTC" />
</v-toolbar-title>
+ <v-spacer />
+ <v-btn
+ :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-refresh' : null"
+ variant="flat"
+ :text="$t('toolbars.table.data.refresh')"
+ class="mr-2"
+ :disabled="loadingSubset"
+ :loading="loadingSubset"
+ @click="loadSubset" />
</v-toolbar>
<v-card tile>
<QueryResults
diff --git a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue
index d52ac91642901fe4b012345c1309235ea4f66420..1d9101fbf240d8d577026e53361f2cedb8f6486e 100644
--- a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue
+++ b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue
@@ -103,7 +103,12 @@
<script setup>
const config = useRuntimeConfig()
const { database_id, subset_id } = useRoute().params
-const { data } = await useFetch(`${config.public.api.server}/api/database/${database_id}/subset/${subset_id}`)
+const requestConfig = { timeout: 90_000, headers: { Accept: 'application/json', 'Content-Type': 'application/json' } }
+const userStore = useUserStore()
+if (userStore.getToken) {
+ requestConfig.headers.Authorization = `Bearer ${userStore.getToken}`
+}
+const { data } = await useFetch(`${config.public.api.server}/api/database/${database_id}/subset/${subset_id}`, requestConfig)
if (data.value) {
const identifierService = useIdentifierService()
useServerHead(identifierService.subsetToServerHead(data.value))
diff --git a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue
index 2d7195fbcdebf9645fc2362f867c4a946e81c2d1..bcab9b60be8812f270836644e5f521553a22040d 100644
--- a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue
+++ b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue
@@ -72,7 +72,7 @@
:headers="headers"
:items="rows"
:items-length="total"
- :loading="loadingData"
+ :loading="loadingData || loadingCount"
:options.sync="options"
:footer-props="footerProps"
@update:options="loadData">
@@ -192,9 +192,6 @@ export default {
}
},
computed: {
- loadingColor () {
- return this.error ? 'error' : 'primary'
- },
roles () {
return this.userStore.getRoles
},
@@ -283,6 +280,7 @@ export default {
},
watch: {
version () {
+ this.loadCount()
this.reload()
},
table (newTable, oldTable) {
@@ -292,8 +290,8 @@ export default {
}
},
mounted () {
- this.reload()
this.loadProperties()
+ this.loadCount()
},
methods: {
addTuple () {
@@ -429,9 +427,20 @@ export default {
reload () {
this.lastReload = new Date()
this.loadData({ page: this.options.page, itemsPerPage: this.options.itemsPerPage, sortBy: null})
- this.loadCount()
},
- loadData ({ page, itemsPerPage, sortBy }) {
+ loadCount() {
+ this.loadingCount = true
+ const tableService = useTableService()
+ tableService.getCount(this.$route.params.database_id, this.$route.params.table_id, (this.versionISO || this.lastReload.toISOString()))
+ .then((count) => {
+ this.total = count
+ this.loadingCount = false
+ })
+ .catch((error) => {
+ this.loadingCount = false
+ })
+ },
+ loadData({ page, itemsPerPage, sortBy }) {
this.options.page = page
this.options.itemsPerPage = itemsPerPage
const tableService = useTableService()
@@ -464,23 +473,6 @@ export default {
toast.error(this.$t(code) + ": " + message)
})
},
- loadCount () {
- const tableService = useTableService()
- this.loadingCount = true
- tableService.getCount(this.$route.params.database_id, this.$route.params.table_id, (this.versionISO || this.lastReload.toISOString()))
- .then((count) => {
- this.total = count
- this.loadingCount = false
- })
- .catch(({code, message}) => {
- this.loadingCount = false
- const toast = useToastInstance()
- if (typeof code !== 'string' || typeof message !== 'string') {
- return
- }
- toast.error(this.$t(code) + ": " + message)
- })
- },
isFileField (column) {
return ['blob', 'longblob', 'mediumblob', 'tinyblob'].includes(column.column_type)
},
diff --git a/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue
index 03464f5dbba66d89312f7b1c4a55332c79af161a..838ef2f0f1adf0a90f722ffd73d43457a5b92186 100644
--- a/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue
+++ b/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue
@@ -10,7 +10,7 @@
:prepend-icon="$vuetify.display.lgAndUp ? 'mdi-refresh' : null"
variant="flat"
:text="$t('toolbars.table.data.refresh')"
- class="mb-1 ml-2"
+ class="mb-1 mr-2"
:loading="loadingData"
@click="reload" />
</v-toolbar>
diff --git a/dbrepo-ui/utils/index.ts b/dbrepo-ui/utils/index.ts
index d940d076ae2b2d4fafac31c6dca8ad833342f64a..995c03a82795545858c477326a57ef93296c4d5b 100644
--- a/dbrepo-ui/utils/index.ts
+++ b/dbrepo-ui/utils/index.ts
@@ -1082,28 +1082,30 @@ export function timestampsToHumanDifference(date1: string, date2: string) {
return moment.duration(other.diff(date)).humanize(true)
}
-export function sizeToHumanLabel(num: number) {
+export function sizeToHumanLabel(num: number): string {
let number = Number(num)
if (!number) {
return '0 B'
}
if (number < 1000) {
- return `${Math.floor(number)} B`
+ return `${roundTwoDecimals(number)} B`
}
number = number / 1000
if (number < 1000) {
- return `${Math.floor(number)} kB`
+ return `${roundTwoDecimals(number)} kB`
}
number = number / 1000
if (number < 1000) {
- return `${Math.floor(number)} MB`
+ return `${roundTwoDecimals(number)} MB`
}
number = number / 1000
if (number < 1000) {
- return `${number} GB`
+ return `${roundTwoDecimals(number)} GB`
}
number = number / 1000
- if (number < 1000) {
- return `${number} TB`
- }
+ return `${roundTwoDecimals(number)} TB`
+}
+
+export function roundTwoDecimals(num: number): number {
+ return Math.round((num + Number.EPSILON) * 100) / 100
}
diff --git a/docker-compose.yml b/docker-compose.yml
index 3ffd0e04a33d4e09e73b4cb8b8a56dfa487283a3..6408cec90431dde9cf7ed97d57d6b30040d9cbf0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -310,6 +310,9 @@ services:
APP_VERSION: ${APP_VERSION:-latest}
COMMIT: ${CI_COMMIT_SHA:-}
network: host
+ environment:
+ NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}"
+ NUXT_PUBLIC_UPLOAD_CLIENT: "${BASE_URL:-http://localhost}/api/upload/files"
depends_on:
dbrepo-search-service:
condition: service_started
diff --git a/helm/dbrepo/templates/auth-configmap.yaml b/helm/dbrepo/templates/auth-configmap.yaml
index 0ef8e90bf95eb5575f2f4fd1628fcf6b8c16e120..269d18c99d7800101afc8ac1c3528104fad7243c 100644
--- a/helm/dbrepo/templates/auth-configmap.yaml
+++ b/helm/dbrepo/templates/auth-configmap.yaml
@@ -2184,7 +2184,7 @@ data:
} ],
"org.keycloak.storage.UserStorageProvider" : [ {
"id" : "c109d473-5ce1-4032-af7b-02e5442f5c07",
- "name" : "openldap",
+ "name" : "Identity Service",
"providerId" : "ldap",
"subComponents" : {
"org.keycloak.storage.ldap.mappers.LDAPStorageMapper" : [ {
@@ -2944,4 +2944,4 @@ data:
"policies" : [ ]
}
}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/helm/dbrepo/templates/metadata-secret.yaml b/helm/dbrepo/templates/metadata-secret.yaml
index ac9fbb32fe149c8d64942ea08424bf7344bd4bf4..0718b02ea1f7ed10d4a53d9d93e04ce985104bb1 100644
--- a/helm/dbrepo/templates/metadata-secret.yaml
+++ b/helm/dbrepo/templates/metadata-secret.yaml
@@ -36,7 +36,7 @@ stringData:
METADATA_HOST: "{{ .Values.metadatadb.host }}"
METADATA_JDBC_EXTRA_ARGS: "{{ .Values.metadatadb.jdbcExtraArgs }}"
METADATA_USERNAME: "{{ .Values.metadatadb.rootUser.user }}"
- METADATA_PASSWORD: "{{ .Values.metadatadb.rootUser.password }}"
+ METADATA_DB_PASSWORD: "{{ .Values.metadatadb.rootUser.password }}"
PID_BASE: "{{ $pidBase }}"
REPOSITORY_NAME: "{{ .Values.metadataservice.repositoryName }}"
ROR_ENDPOINT: "{{ .Values.metadataservice.ror.endpoint}}"
diff --git a/helm/dbrepo/templates/search-db-secret.yaml b/helm/dbrepo/templates/search-db-secret.yaml
deleted file mode 100644
index 81cc79db7c68449bd4b416863db000c818ec59a7..0000000000000000000000000000000000000000
--- a/helm/dbrepo/templates/search-db-secret.yaml
+++ /dev/null
@@ -1,79 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-type: kubernetes.io/tls
-metadata:
- name: search-db-secret
- namespace: {{ .Values.namespace }}
-data:
- tls.crt: |
- LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM0akNDQWNxZ0F3SUJBZ0lSQVAvcFJoaFQ5
- SFVWaUFzYitybmJjdkV3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSmMyVmhj
- bU5vTFdSaU1CNFhEVEkwTURRd056RTRORFEwT0ZvWERUSTBNRGN3TmpFNApORFEwT0Zvd0ZERVNN
- QkFHQTFVRUF4TUpjMlZoY21Ob0xXUmlNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFN
- SUlCQ2dLQ0FRRUFzb3lTWTduV3J0MTRVQjNST0kwOWRtMVNSU2lDZ2hQYVhwRlJTMjhjalpNSUFz
- TUoKR2ZwREZ5VktOQ3pTV0VZN0J2M1JpOHlrRnlZNkpFb2p0S3oxdk9GNnNyQ2JYZnhsY1NiZDk3
- UVYwdU9IYTZKRApsWGN2aUJEKzN2ZTB0K0MzRGFPSFVMY1liVWkzS2xOS3FwTDU1Q2ZNeTYzdU4z
- a21zekRwTjVycWhOYnBlVTAxCnd2NFZNaldNZ3RlU1VpWDNqeU1EcUFOa1B3UXFiYnZHN0hBUm54
- Q0QvMHJFeEVvRjNqRCtGV01XbEVjdXR1VGkKbFJ5QnN3L1FLTWd0aVJVSFJXYUJGK2ZES0wxSUoz
- YVhmcDR5bmNhL2tCR3pxVGpqb3dJb2R0MEdOZjBFa1QyQgpTWG9hZGtwdVptT2JiVDF2UmN1T1BH
- UEZjWVd5Qm1ucS9adEorUUlEQVFBQm95OHdMVEFkQmdOVkhTVUVGakFVCkJnZ3JCZ0VGQlFjREFR
- WUlLd1lCQlFVSEF3SXdEQVlEVlIwVEFRSC9CQUl3QURBTkJna3Foa2lHOXcwQkFRc0YKQUFPQ0FR
- RUFlUU4vaUsvRzhHbGt5R0w1NjlrZnBiWEE2bE8vRHFObGlXRkgrY2ZIZ0NzYWxKMWVSSjliY1RZ
- dgo0S3Y0MDlWUWpCbVg0WTRqMUt6R1ZnYkZaZkh1Ry9Nb0dzWVVnQ1VjTm94ZThtM0ZUcjRwYnZT
- MXNUV0V4cGFNCkpSMURQQmNMV0o3MndTQzBkRFpISC9hVVNSMUs4UGpnMWtaMVRINTdvZDJoNWpJ
- RUFhZkd1ZGhzejVpWlZQcVkKR1lrakZhRklVeXpjWkxUbjFBNXRwSlpTRmhxZHZGQmFndURUYkp4
- NmROVWZVc0sxZXFuaThSQVN6L3dPbHQwcQpSckExbVdCTEI1NW9XRzh4ZXZicmtNNUNuSWVvL2hS
- SG83cE1pUFQxWE5uT2cvNjhmZEc4T0lXMFFhNjdMVEZnCnU2dTkxQ1BmVk5KVHQ5bmlZWHJ4N1hl
- SEJ2dW1iUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
- ca.crt: |
- LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM0akNDQWNxZ0F3SUJBZ0lSQVAvcFJoaFQ5
- SFVWaUFzYitybmJjdkV3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSmMyVmhj
- bU5vTFdSaU1CNFhEVEkwTURRd056RTRORFEwT0ZvWERUSTBNRGN3TmpFNApORFEwT0Zvd0ZERVNN
- QkFHQTFVRUF4TUpjMlZoY21Ob0xXUmlNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFN
- SUlCQ2dLQ0FRRUFzb3lTWTduV3J0MTRVQjNST0kwOWRtMVNSU2lDZ2hQYVhwRlJTMjhjalpNSUFz
- TUoKR2ZwREZ5VktOQ3pTV0VZN0J2M1JpOHlrRnlZNkpFb2p0S3oxdk9GNnNyQ2JYZnhsY1NiZDk3
- UVYwdU9IYTZKRApsWGN2aUJEKzN2ZTB0K0MzRGFPSFVMY1liVWkzS2xOS3FwTDU1Q2ZNeTYzdU4z
- a21zekRwTjVycWhOYnBlVTAxCnd2NFZNaldNZ3RlU1VpWDNqeU1EcUFOa1B3UXFiYnZHN0hBUm54
- Q0QvMHJFeEVvRjNqRCtGV01XbEVjdXR1VGkKbFJ5QnN3L1FLTWd0aVJVSFJXYUJGK2ZES0wxSUoz
- YVhmcDR5bmNhL2tCR3pxVGpqb3dJb2R0MEdOZjBFa1QyQgpTWG9hZGtwdVptT2JiVDF2UmN1T1BH
- UEZjWVd5Qm1ucS9adEorUUlEQVFBQm95OHdMVEFkQmdOVkhTVUVGakFVCkJnZ3JCZ0VGQlFjREFR
- WUlLd1lCQlFVSEF3SXdEQVlEVlIwVEFRSC9CQUl3QURBTkJna3Foa2lHOXcwQkFRc0YKQUFPQ0FR
- RUFlUU4vaUsvRzhHbGt5R0w1NjlrZnBiWEE2bE8vRHFObGlXRkgrY2ZIZ0NzYWxKMWVSSjliY1RZ
- dgo0S3Y0MDlWUWpCbVg0WTRqMUt6R1ZnYkZaZkh1Ry9Nb0dzWVVnQ1VjTm94ZThtM0ZUcjRwYnZT
- MXNUV0V4cGFNCkpSMURQQmNMV0o3MndTQzBkRFpISC9hVVNSMUs4UGpnMWtaMVRINTdvZDJoNWpJ
- RUFhZkd1ZGhzejVpWlZQcVkKR1lrakZhRklVeXpjWkxUbjFBNXRwSlpTRmhxZHZGQmFndURUYkp4
- NmROVWZVc0sxZXFuaThSQVN6L3dPbHQwcQpSckExbVdCTEI1NW9XRzh4ZXZicmtNNUNuSWVvL2hS
- SG83cE1pUFQxWE5uT2cvNjhmZEc4T0lXMFFhNjdMVEZnCnU2dTkxQ1BmVk5KVHQ5bmlZWHJ4N1hl
- SEJ2dW1iUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
- tls.key: |
- LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZB
- QVNDQktjd2dnU2pBZ0VBQW9JQkFRQ3lqSkpqdWRhdTNYaFEKSGRFNGpUMTJiVkpGS0lLQ0U5cGVr
- VkZMYnh5Tmt3Z0N3d2taK2tNWEpVbzBMTkpZUmpzRy9kR0x6S1FYSmpvawpTaU8wclBXODRYcXlz
- SnRkL0dWeEp0MzN0QlhTNDRkcm9rT1ZkeStJRVA3ZTk3UzM0TGNObzRkUXR4aHRTTGNxClUwcXFr
- dm5rSjh6THJlNDNlU2F6TU9rM211cUUxdWw1VFRYQy9oVXlOWXlDMTVKU0pmZVBJd09vQTJRL0JD
- cHQKdThic2NCR2ZFSVAvU3NURVNnWGVNUDRWWXhhVVJ5NjI1T0tWSElHekQ5QW95QzJKRlFkRlpv
- RVg1OE1vdlVnbgpkcGQrbmpLZHhyK1FFYk9wT09PakFpaDIzUVkxL1FTUlBZRkplaHAyU201bVk1
- dHRQVzlGeTQ0OFk4VnhoYklHCmFlcjltMG41QWdNQkFBRUNnZ0VCQUtPZ3A5ZTB5OFhkT1JGVEFo
- WXRlaEk2QlpkVGxLYll3dHEvbWh6amF1dGoKdjRlb2JZTGRFdmIzT1pXdkxlV3dGeEJGTS9CR1Rt
- cllvWmY0U2RpZVdXWUx6WUpNejFYR3BNQ1p1Zm56azd4OAp2L0luOW4vWGhqdlFONExteHp0c09O
- WEs4NHRKQUozR2NmWGI5eVZ6SklldTRjUVhWYVNJNXFwNVBJRzArdzlZCnk2NTFWZkZJQUd3SmRI
- QlpId1lmQUdxbU5oVlo3MDc4TVUxQWU2Y2VkZjJ0RnlWYW5ScXBLUFZ1Z0tGQy9kRG8KVXJIMHRJ
- ajFkU3RKRGxucHJ3YVYrMDRkUDBvZnlBc09ablp3VXRzZE8vM1ZMMWR4bCtIT1dGeUpvSjI1dkF2
- eAp5ZW5qc0dzd1pJRW1oUzM0NVRVTlFNbTJzYnJtYklMS1dpWEp5SmlEeWdFQ2dZRUF4QUNEbWxG
- ZmsxZDlkSmJTCi90NDRGTzFiUVQ2b1Z2VWJ2NGMwcnRLVVhwTnFlajNmbXl4bUJINm82SkhoYjNO
- eXdWa3U0QW9YTXBFTkhISDAKNlN2TzBYM2U1MU8xOXppQ25hMGdoZExSS3JIS3ZqbzRNdmdiV3Ey
- Z3NJNmpJQkxTcU0zNXNxTjhGRXRVZko4TQpKRUZIMThJTThTRjEyQ2hWYjVWaThTTDgvY2tDZ1lF
- QTZUUmhVQi9nQVM1RmN5V1NHNGR4UWtHWG03R1lSamFrCjJVWUlOUUIyV2d2Qk9vN2tvV21nR1M0
- eE9YanJsZ2NrMmhsTEhZSXAycnRoSWdNMUdBQUdqa2lYeXJPVE9kaGQKeUJ0RjBMS0kvVjlBZUxK
- eExkRDBYalB0WThIYXdTcW8weGdxUml2RzlJUFBlSGZ6SmlraXV6enNOT3pjVlk1aApkZktqZy9J
- eVFyRUNnWUJ1c0dlaDk4Q0ZBa3pNVWZ6NGlHQ2RtT29ITDY1NzVWSjFXSkx0QStsY2U5NFBDUEJG
- CnZzNGlUYkZ3SGlwMCtYcmVMRkpubmVzNTJHYlNJSjBTTFhaUUlzaUdWV1VYSjZmRUNpaXF5c0xy
- WEpyRjBUVTUKdTVvZkhKejUrS094RWxBN21vOGdUbWxkUUttRzgzODAzbFVIU1FSc0ROeHpaVnZT
- ZDBmNExDMDUyUUtCZ0h0YQpzNmJZVlhzS2FMNFJ2NGxFU1lxTWU0OWxqM0NFY3dwaTJ2QitRQnc5
- WDRhRUV6ZTJVWE5BVmRWYXV2THU4SFZWCkw4QjZHMzJSNUQxRGlSQWE0MXpiMVQ3cFloVVU5L1pq
- UnJpdjEzcCtxZkd1SWVQa1JYNlc1UmtCYjU4QjI2OWQKZHU4TE5RQWR3TjZ1UkRXSlNNL1YxL1Bl
- M21WN0hONXc3RUZkR1d6aEFvR0FiN29HOWhoTGxTL212bEhHVE11egp6SXhtSTBaTisyZGFqNEYy
- MDZHRGJYaUkwYUtFTHlteVJkcEZYcStzaWpaTzFqdXU4WDhMS0FCOUVaeEtXRzRICjNnQWoxWGFF
- QXFjNjFkaFkveHRHUDR2OXN3UElmVjR3NGtTUHBWYlFFd3ZIcko4K1V2NjhuOTZTU3dkK1BBNisK
- OFFwUEVYSXlscXB4UlFyMnN5amhoOTQ9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
\ No newline at end of file
diff --git a/helm/dbrepo/templates/search-secret.yaml b/helm/dbrepo/templates/search-secret.yaml
index 41665ac2bc1614653262f93cf28882a55638e4ec..251da00248f77afa295483890064ed6a3bc73211 100644
--- a/helm/dbrepo/templates/search-secret.yaml
+++ b/helm/dbrepo/templates/search-secret.yaml
@@ -13,7 +13,7 @@ stringData:
AUTH_SERVICE_CLIENT: "{{ .Values.authservice.client.id }}"
AUTH_SERVICE_CLIENT_SECRET: "{{ .Values.authservice.client.secret }}"
AUTH_SERVICE_ENDPOINT: "{{ .Values.authservice.endpoint }}"
- GATEWAY_SERVICE_ENDPOINT: "{{ .Values.gateway }}"
+ METADATA_SERVICE_ENDPOINT: "{{ .Values.metadataservice.endpoint }}"
JWT_PUBKEY: "{{ .Values.authservice.jwt.pubkey }}"
LOG_LEVEL: "{{ ternary "DEBUG" "INFO" .Values.searchservice.image.debug }}"
OPENSEARCH_HOST: "{{ .Values.searchdb.host }}"
diff --git a/helm/dbrepo/values.yaml b/helm/dbrepo/values.yaml
index 2b18ed14222517593b1d9a31ec3bf67633fd2e91..af810436af63b6ca3b87a08c767a28959852ec35 100644
--- a/helm/dbrepo/values.yaml
+++ b/helm/dbrepo/values.yaml
@@ -56,8 +56,8 @@ metadatadb:
enabled: false
## @skip metadatadb.initdbScriptsConfigMap The initial database scripts.
initdbScriptsConfigMap: metadata-db-setup
- ## @param metadatadb.initdbScripts Additional init.db scripts that are executed on the first start.
- initdbScripts: { }
+ ## @param metadatadb.extraInitDbScripts Additional init.db scripts that are executed on the first start.
+ extraInitDbScripts: { }
# 03-additional-data.sql: |
# BEGIN;
# INSERT INTO `mdb_containers` (name, internal_name, image_id, host, port, sidecar_host, sidecar_port, privileged_username, privileged_password)
@@ -81,11 +81,6 @@ authservice:
debug: false
## @param authservice.endpoint The hostname for the microservices.
endpoint: http://auth-service
- auth:
- ## @param authservice.auth.adminUser The admin username.
- adminUser: fda
- ## @param authservice.auth.adminPassword The admin user password.
- adminPassword: fda
## @skip authservice.postgresql
postgresql:
enabled: true
@@ -222,8 +217,6 @@ searchdb:
## @skip searchdb.security
security:
enabled: false
- adminUsername: admin
- adminPassword: admin
## @param searchdb.clusterName The cluster name.
clusterName: search-db
@@ -422,7 +415,7 @@ metadataservice:
## @param metadataservice.podSecurityContext.fsGroup Set RabbitMQ pod's Security Context fsGroup
fsGroup: 1001
containerSecurityContext:
- ## @param metadataservice.containerSecurityContext.enabled Enabled containers' Security Context
+ ## @param metadataservice.containerSecurityContext.enabled Enable containers' Security Context
enabled: true
## @param metadataservice.containerSecurityContext.seLinuxOptions Set SELinux options in container
seLinuxOptions: { }
@@ -812,14 +805,14 @@ ui:
host: example.com
port:
## @param ui.public.broker.port.5671 Enable display of the broker 5671 port and mark it as secure (SSL/TLS).
- 5671: true
+ 5671: false
## @param ui.public.broker.port.5672 Enable display of the broker 5672 port and mark it as insecure (no SSL/TLS).
- 5672: false
+ 5672: true
## @param ui.public.broker.extra Extra metadata displayed.
extra: ""
database:
## @param ui.public.database.extra Extra metadata displayed.
- extra: "128.130.0.0/15"
+ extra: ""
## @skip ui.public.links
links:
rabbitmq:
diff --git a/install.sh b/install.sh
index a9fcf6d10d374f048d14289d20fc89d01ef82813..5e367f4d53fefc633fb3131dad7360569ea971ac 100644
--- a/install.sh
+++ b/install.sh
@@ -69,15 +69,15 @@ fi
echo "[📦] Pulling images for version ${VERSION} ..."
docker compose pull
-echo "[✨] Starting DBRepo ..."
-docker compose up -d
-
-if [ $? -eq 0 ]; then
- echo "[🎉] Successfully started!"
- echo ""
- echo "You can now inspect the logs with:"
- echo ""
- echo " docker compose logs -f"
- echo ""
- echo "Read about next steps online: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${VERSION}/installation/#next-steps"
-fi
+echo "[🎉] Success!"
+echo ""
+echo "You can now:"
+echo ""
+echo " 1) Either start the deployment running on http://localhost, or"
+echo " 2) Edit the BASE_URL variable in .env to set your hostname"
+echo ""
+echo "Then start the local deployment with:"
+echo ""
+echo " docker compose up -d"
+echo ""
+echo "Read about next steps online: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${VERSION}/installation/#next-steps"
diff --git a/mkdocs.yml b/mkdocs.yml
index 9bb4588f27727a8e7926caf585094989553957fa..5285953568a544a7589bb091e82c13138f6b84cb 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -47,10 +47,12 @@ nav:
- Air Quality Data: examples/air.md
- COVID-19 Data: examples/covid-19.md
- Hazard Data: examples/hazard.md
+ - Health Data: examples/health.md
- Industry 4.0 Power Data: examples/power.md
- Survey Data: examples/survey.md
- Lute Data: examples/lute-data.md
- Music-ML Data: examples/music.md
+ - Theater Data: examples/theater.md
- Transportation Data: examples/transportation.md
- XPS Data: examples/xps-data.md
- publications.md