diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
index 13cfc5c56011512f6137bb728933fe5f41055c35..64a5a4546cc5c0b75bc611613d6b3327669fc1e2 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java
@@ -10,10 +10,7 @@ import at.tuwien.api.database.query.QueryPersistDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
import at.tuwien.mapper.MetadataMapper;
-import at.tuwien.service.CredentialService;
-import at.tuwien.service.SchemaService;
-import at.tuwien.service.StorageService;
-import at.tuwien.service.SubsetService;
+import at.tuwien.service.*;
import at.tuwien.utils.UserUtil;
import at.tuwien.validation.EndpointValidator;
import io.micrometer.observation.annotation.Observed;
@@ -55,17 +52,19 @@ public class SubsetEndpoint extends AbstractEndpoint {
private final SchemaService schemaService;
private final SubsetService subsetService;
private final MetadataMapper metadataMapper;
+ private final MetricsService metricsService;
private final StorageService storageService;
private final CredentialService credentialService;
private final EndpointValidator endpointValidator;
@Autowired
public SubsetEndpoint(SchemaService schemaService, SubsetService subsetService, MetadataMapper metadataMapper,
- StorageService storageService, CredentialService credentialService,
- EndpointValidator endpointValidator) {
+ MetricsService metricsService, StorageService storageService,
+ CredentialService credentialService, EndpointValidator endpointValidator) {
this.schemaService = schemaService;
this.subsetService = subsetService;
this.metadataMapper = metadataMapper;
+ this.metricsService = metricsService;
this.storageService = storageService;
this.credentialService = credentialService;
this.endpointValidator = endpointValidator;
@@ -366,6 +365,7 @@ public class SubsetEndpoint extends AbstractEndpoint {
.build();
}
final Dataset<Row> dataset = subsetService.getData(database, subset, page, size);
+ metricsService.countSubsetGetData(databaseId, subsetId);
final ViewDto view = schemaService.inspectView(database, metadataMapper.queryDtoToViewName(subset));
headers.set("Access-Control-Expose-Headers", "X-Id X-Headers");
headers.set("X-Headers", String.join(",", view.getColumns().stream().map(ViewColumnDto::getInternalName).toList()));
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
index 4191726dd9a5b10cf26055bbe91c323c73e75450..831e16099d4b79af7fc3ab963879036c34efbc1b 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
@@ -12,10 +12,7 @@ import at.tuwien.api.database.table.internal.TableCreateDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
-import at.tuwien.service.CredentialService;
-import at.tuwien.service.SchemaService;
-import at.tuwien.service.StorageService;
-import at.tuwien.service.TableService;
+import at.tuwien.service.*;
import at.tuwien.utils.UserUtil;
import at.tuwien.validation.EndpointValidator;
import io.micrometer.observation.annotation.Observed;
@@ -55,17 +52,19 @@ public class TableEndpoint extends AbstractEndpoint {
private final TableService tableService;
private final SchemaService schemaService;
+ private final MetricsService metricsService;
private final StorageService storageService;
private final CredentialService credentialService;
private final EndpointValidator endpointValidator;
private final MetadataServiceGateway metadataServiceGateway;
@Autowired
- public TableEndpoint(TableService tableService, SchemaService schemaService, StorageService storageService,
- CredentialService credentialService, EndpointValidator endpointValidator,
- MetadataServiceGateway metadataServiceGateway) {
+ public TableEndpoint(TableService tableService, SchemaService schemaService, MetricsService metricsService,
+ StorageService storageService, CredentialService credentialService,
+ EndpointValidator endpointValidator, MetadataServiceGateway metadataServiceGateway) {
this.tableService = tableService;
this.schemaService = schemaService;
+ this.metricsService = metricsService;
this.storageService = storageService;
this.credentialService = credentialService;
this.endpointValidator = endpointValidator;
@@ -291,10 +290,12 @@ public class TableEndpoint extends AbstractEndpoint {
}
headers.set("Access-Control-Expose-Headers", "X-Headers");
headers.set("X-Headers", String.join(",", table.getColumns().stream().map(ColumnDto::getInternalName).toList()));
+ final Dataset<Row> dataset = tableService.getData(table.getDatabase(), table.getInternalName(), timestamp,
+ null, null, null, null);
+ metricsService.countTableGetData(databaseId, tableId);
return ResponseEntity.ok()
.headers(headers)
- .body(transform(tableService.getData(table.getDatabase(), table.getInternalName(), timestamp,
- null, null, null, null)));
+ .body(transform(dataset));
} catch (SQLException | QueryMalformedException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
index b08c300b450c4035d7fdd5975503a67afe9ff414..3b881488ba736821628eb8b9bca263b35b8a4074 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java
@@ -8,10 +8,7 @@ import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.internal.PrivilegedViewDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.exception.*;
-import at.tuwien.service.CredentialService;
-import at.tuwien.service.StorageService;
-import at.tuwien.service.TableService;
-import at.tuwien.service.ViewService;
+import at.tuwien.service.*;
import at.tuwien.utils.UserUtil;
import at.tuwien.validation.EndpointValidator;
import io.micrometer.observation.annotation.Observed;
@@ -26,6 +23,8 @@ import jakarta.servlet.http.HttpServletRequest;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;
import lombok.extern.log4j.Log4j2;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpHeaders;
@@ -48,15 +47,18 @@ public class ViewEndpoint extends AbstractEndpoint {
private final ViewService viewService;
private final TableService tableService;
+ private final MetricsService metricsService;
private final StorageService storageService;
private final CredentialService credentialService;
private final EndpointValidator endpointValidator;
@Autowired
- public ViewEndpoint(ViewService viewService, TableService tableService, StorageService storageService,
- CredentialService credentialService, EndpointValidator endpointValidator) {
+ public ViewEndpoint(ViewService viewService, TableService tableService, MetricsService metricsService,
+ StorageService storageService, CredentialService credentialService,
+ EndpointValidator endpointValidator) {
this.viewService = viewService;
this.tableService = tableService;
+ this.metricsService = metricsService;
this.storageService = storageService;
this.credentialService = credentialService;
this.endpointValidator = endpointValidator;
@@ -286,10 +288,12 @@ public class ViewEndpoint extends AbstractEndpoint {
}
headers.set("Access-Control-Expose-Headers", "X-Headers");
headers.set("X-Headers", String.join(",", view.getColumns().stream().map(ViewColumnDto::getInternalName).toList()));
+ final Dataset<Row> dataset = tableService.getData(view.getDatabase(), view.getInternalName(), timestamp,
+ page, size, null, null);
+ metricsService.countViewGetData(databaseId, viewId);
return ResponseEntity.ok()
.headers(headers)
- .body(transform(tableService.getData(view.getDatabase(), view.getInternalName(), timestamp, page,
- size, null, null)));
+ .body(transform(dataset));
} catch (SQLException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java b/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java
index a77af353d3b48f84706bbfff80cdcde1f374b414..45654157d1a7564d6504e08bd6d6d5fa5d24b206 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/config/CacheConfig.java
@@ -1,25 +1,64 @@
package at.tuwien.config;
-import at.tuwien.api.PrivilegedObjectDto;
+import at.tuwien.api.container.internal.PrivilegedContainerDto;
+import at.tuwien.api.database.DatabaseAccessDto;
+import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
+import at.tuwien.api.database.internal.PrivilegedViewDto;
+import at.tuwien.api.database.table.internal.PrivilegedTableDto;
+import at.tuwien.api.user.internal.PrivilegedUserDto;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
@Configuration
-public class CacheConfig<K, T extends PrivilegedObjectDto> {
+public class CacheConfig {
@Value("${dbrepo.credentialCacheTimeout}")
private Long credentialCacheTimeout;
@Bean
- public Cache<K, T> cache() {
- return Caffeine.newBuilder()
- .expireAfterWrite(credentialCacheTimeout, TimeUnit.SECONDS)
- .build();
+ public Cache<UUID, PrivilegedUserDto> userCache() {
+ return new ExpiryCache<UUID, PrivilegedUserDto>().build();
+ }
+
+ @Bean
+ public Cache<Long, PrivilegedViewDto> viewCache() {
+ return new ExpiryCache<Long, PrivilegedViewDto>().build();
+ }
+
+ @Bean
+ public Cache<Long, DatabaseAccessDto> accessCache() {
+ return new ExpiryCache<Long, DatabaseAccessDto>().build();
+ }
+
+ @Bean
+ public Cache<Long, PrivilegedTableDto> tableCache() {
+ return new ExpiryCache<Long, PrivilegedTableDto>().build();
+ }
+
+ @Bean
+ public Cache<Long, PrivilegedDatabaseDto> databaseCache() {
+ return new ExpiryCache<Long, PrivilegedDatabaseDto>().build();
+ }
+
+ @Bean
+ public Cache<Long, PrivilegedContainerDto> containerCache() {
+ return new ExpiryCache<Long, PrivilegedContainerDto>().build();
+ }
+
+ class ExpiryCache<K, T> {
+
+ Cache<K, T> build() {
+ return Caffeine.newBuilder()
+ .expireAfterWrite(credentialCacheTimeout, TimeUnit.SECONDS)
+ .build();
+ }
+
}
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java b/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java
index 9ff09ab42b2c2e62536e1d46fa986057ff664d53..450be2f7df8b52fe493dd498dc0422350bb3ff39 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/config/MetricsConfig.java
@@ -1,7 +1,5 @@
package at.tuwien.config;
-import io.micrometer.core.instrument.Counter;
-import io.micrometer.core.instrument.Metrics;
import io.micrometer.observation.ObservationRegistry;
import io.micrometer.observation.aop.ObservedAspect;
import org.springframework.context.annotation.Bean;
@@ -14,20 +12,4 @@ public class MetricsConfig {
public ObservedAspect observedAspect(ObservationRegistry observationRegistry) {
return new ObservedAspect(observationRegistry);
}
-
- @Bean
- public Counter httpDataAccessCounter() {
- return Counter.builder("dbrepo.data.access")
- .tag("protocol", "http")
- .description("The total number of accessed data sources")
- .register(Metrics.globalRegistry);
- }
-
- @Bean
- public Counter amqpDataAccessCounter() {
- return Counter.builder("dbrepo.data.access")
- .tag("protocol", "amqp")
- .description("The total number of accessed data sources")
- .register(Metrics.globalRegistry);
- }
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/MetricsService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/MetricsService.java
new file mode 100644
index 0000000000000000000000000000000000000000..c0e41ec9f030d7c78d8ff92d4be25dd1a954dd9b
--- /dev/null
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/MetricsService.java
@@ -0,0 +1,12 @@
+package at.tuwien.service;
+
+public interface MetricsService {
+
+ void countDatabaseGetData(Long databaseId);
+
+ void countTableGetData(Long databaseId, Long tableId);
+
+ void countSubsetGetData(Long databaseId, Long subsetId);
+
+ void countViewGetData(Long databaseId, Long viewId);
+}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/MetricsServicePrometheusImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/MetricsServicePrometheusImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..8916cc48a90b796fca6786a2fbcc8891ea0179ed
--- /dev/null
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/MetricsServicePrometheusImpl.java
@@ -0,0 +1,56 @@
+package at.tuwien.service.impl;
+
+import at.tuwien.service.MetricsService;
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Metrics;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Service;
+
+@Log4j2
+@Service
+public class MetricsServicePrometheusImpl implements MetricsService {
+
+ @Override
+ public void countDatabaseGetData(Long databaseId) {
+ Counter.builder("dbrepo.database.data.get")
+ .tag("database_id", "" + databaseId)
+ .tag("protocol", "http")
+ .description("The total number of accessed data sources")
+ .register(Metrics.globalRegistry)
+ .increment();
+ }
+
+ @Override
+ public void countTableGetData(Long databaseId, Long tableId) {
+ Counter.builder("dbrepo.table.data.get")
+ .tag("database_id", "" + databaseId)
+ .tag("table_id", "" + tableId)
+ .tag("protocol", "http")
+ .description("The total number of accessed data sources")
+ .register(Metrics.globalRegistry)
+ .increment();
+ }
+
+ @Override
+ public void countSubsetGetData(Long databaseId, Long subsetId) {
+ Counter.builder("dbrepo.table.data.get")
+ .tag("database_id", "" + databaseId)
+ .tag("subset_id", "" + subsetId)
+ .tag("protocol", "http")
+ .description("The total number of accessed data sources")
+ .register(Metrics.globalRegistry)
+ .increment();
+ }
+
+ @Override
+ public void countViewGetData(Long databaseId, Long viewId) {
+ Counter.builder("dbrepo.table.data.get")
+ .tag("database_id", "" + databaseId)
+ .tag("view_id", "" + viewId)
+ .tag("protocol", "http")
+ .description("The total number of accessed data sources")
+ .register(Metrics.globalRegistry)
+ .increment();
+ }
+
+}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
index aff85137870749a600d1e4c76ef317a978c6c54e..d5127e050e521b69e5130c4af836680ffaae013d 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java
@@ -6,7 +6,6 @@ import at.tuwien.mapper.DataMapper;
import at.tuwien.mapper.MetadataMapper;
import at.tuwien.service.QueueService;
import com.mchange.v2.c3p0.ComboPooledDataSource;
-import io.micrometer.core.instrument.Counter;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -21,14 +20,11 @@ import java.util.Optional;
@Service
public class QueueServiceRabbitMqImpl extends HibernateConnector implements QueueService {
- private final Counter amqpDataAccessCounter;
private final DataMapper dataMapper;
private final MetadataMapper metadataMapper;
@Autowired
- public QueueServiceRabbitMqImpl(Counter amqpDataAccessCounter, DataMapper dataMapper,
- MetadataMapper metadataMapper) {
- this.amqpDataAccessCounter = amqpDataAccessCounter;
+ public QueueServiceRabbitMqImpl(DataMapper dataMapper, MetadataMapper metadataMapper) {
this.dataMapper = dataMapper;
this.metadataMapper = metadataMapper;
}
@@ -54,7 +50,6 @@ public class QueueServiceRabbitMqImpl extends HibernateConnector implements Queu
preparedStatement.executeUpdate();
log.trace("executed statement in {} ms", System.currentTimeMillis() - start);
log.trace("successfully inserted tuple");
- amqpDataAccessCounter.increment();
} finally {
dataSource.close();
}