Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
Loading items

Target

Select target project
  • fair-data-austria-db-repository/fda-services
1 result
Select Git revision
Loading items
Show changes
Showing
with 753 additions and 191 deletions
import logging
import os
from dbrepo.api.dto import Database, View
from clients import grafana_client
statistics_row_title = '${view_id}'
base_url = os.getenv('BASE_URL', 'http://localhost')
datasource_uid = os.getenv('JSON_DATASOURCE_NAME', 'dbrepojson0')
def map_link(title: str, url: str) -> dict:
return dict(targetBlank=True,
asDropdown=False,
includeVars=False,
keepTime=False,
tags=[],
type='link',
icon='info',
title=title,
url=url)
def map_statistics_row(dashboard: dict) -> dict | None:
filtered_panels = [panel for panel in dashboard['panels'] if
panel['type'] == 'row' and panel['title'] == statistics_row_title]
if len(filtered_panels) == 0:
logging.warning(f"Failed to find statistics row title {statistics_row_title} in: {filtered_panels}")
return None
return filtered_panels[0]
def map_links(database: Database) -> [dict]:
links = []
if len(database.identifiers) > 0:
links.append(map_link('Database', f"{base_url}/pid/{database.identifiers[0].id}"))
else:
links.append(map_link('Database', f"{base_url}/database/{database.id}"))
return links
def map_templating(database: Database) -> dict:
options = [dict(selected=False,
text=view.name,
value=str(view.id)) for view in database.views]
selected = dict(selected=True,
text=[view.name for view in database.views],
value=[str(view.id) for view in database.views])
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(list=[dict(description='',
name='view_id',
hide=0,
includeAll=True,
multi=True,
datasource=datasource,
refresh=1,
regex='',
sort=0,
definition='dbrepo-json- (infinity) json',
query=dict(queryType='infinity',
query='',
infinityQuery=dict(format='table',
filters=[],
parser='backend',
refId='variable',
root_selector='',
source='url',
type='json',
url=f"/api/database/{database.id}/view",
columns=[dict(selector='id',
text='value',
type='string'),
dict(
selector='internal_name',
text='name',
type='string')],
url_options=dict(data='',
method='GET'))),
label='Datasource',
skipUrlSync=False,
type='query',
current=selected,
options=options)])
def map_timeseries_panel(database: Database, view: View) -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(
title=view['name'],
type='timeseries',
datasource=datasource,
targets=[dict(datasource=datasource,
format='table',
global_query_id='',
hide=False,
refId='A',
root_selector='',
source='url',
type='json',
url=f"/api/database/{database['id']}/view/{view['id']}",
url_options=dict(data='',
method='GET'))],
gridPos=dict(h=8,
w=12,
x=0,
y=0),
options=dict(legend=dict(displayMode='list',
placement='bottom',
showLegend=True),
tooltip=dict(mode='single',
sort='none')),
fieldConfig=dict(
defaults=dict(color=dict(mode='palette-classic'),
custom=dict(
axisBorderShow=False,
axisCenteredZero=False,
axisColorMode='text',
axisLabel='',
axisPlacement='auto',
barAlignment=0,
drawStyle='line',
fillOpacity=0,
gradientMode='none',
hideFrom=dict(legend=False,
tooltip=False,
viz=False),
insertNulls=False,
lineInterpolation='linear',
lineWidth=1,
pointSize=5,
scaleDistribution=dict(type='linear'),
showPoints='auto',
spanNulls=False,
stacking=dict(group='A',
mode='none'),
thresholdsStyle=dict(mode='absolute')))))
def map_statistics_panel(database_id: str, view: View) -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(
title=view.name,
type='table',
datasource=datasource,
targets=[dict(datasource=datasource,
columns=[],
filters=[],
format='table',
global_query_id='',
hide=False,
refId='A',
root_selector='',
source='url',
type='json',
url=f"/api/database/{database_id}/view/{view.id}/data",
url_options=dict(data='',
method='GET'))],
options=dict(cellHeight="sm",
showHeader=True,
footer=dict(countRows=False,
fields="",
reducer=["sum"],
show=False)),
gridPos=dict(h=8,
w=12,
x=12,
y=0),
transformations=dict(id="organize",
options=dict(excludeByName=dict(),
includeByName=dict(),
indexByName=dict(
HEADER_AVG=3,
HEADER_COL=0,
HEADER_STDDEV=4,
HEADER_MAX=2,
HEADER_MIN=1))),
fieldConfig=dict(defaults=dict(custom=dict(align="auto",
filterable="true",
cellOptions=dict(type="auto"),
inspect=False),
mappings=[],
thresholds=dict(mode="absolute",
steps=[dict(color="green",
value=None),
dict(color="red",
value=80)
])),
overrides=[dict(matcher=dict(id="byName",
options="HEADER_COL"),
properties=[dict(id="custom.align",
value="center")]),
dict(matcher=dict(id="byName",
options="HEADER_MIN"),
properties=[dict(id="custom.width",
value=115)]),
dict(matcher=dict(id="byName",
options="HEADER_MAX"),
properties=[dict(id="custom.width",
value=115)]),
dict(matcher=dict(id="byName",
options="HEADER_AVG"),
properties=[dict(id="custom.width",
value=115)]),
dict(matcher=dict(id="byName",
options="HEADER_STDDEV"),
properties=[dict(id="custom.width",
value=115)])
]))
def map_overview_panel(database_id: str) -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(title='Preview',
type='table',
fieldConfig=dict(
defaults=dict(
color=dict(mode='palette-classic'),
custom=dict(axisBorderShow=False,
axisCenteredZero=False,
axisColorMode='text',
axisLabel='',
axisPlacement='auto',
barAlignment=0,
drawStyle='line',
fillOpacity=0,
gradientMode='none',
hideFrom=dict(
legend=False,
tooltip=False,
viz=False),
insertNulls=False,
lineInterpolation='linear',
lineWidth=1,
pointSize=5,
scaleDistribution=dict(
type='linear'),
showPoints='auto',
spanNulls=False,
stacking=dict(group='A',
mode='none'),
thresholdsStyle=dict(
mode='off'))),
overrides=[]),
options=dict(legend=dict(displayMode='list',
placement='bottom',
showLegend=True,
calcs=[]),
tooltip=dict(mode='single',
sort='none')),
targets=[dict(format='json',
columns=[],
datasource=datasource,
filters=[],
global_query_id='',
refId='A',
root_selector='',
source='url',
type='json',
url='/api/database/' + database_id + '/view/${view_id}/data',
url_options=dict(data='',
method='GET'))],
datasource=datasource,
gridPos=dict(h=4,
w=12,
x=0,
y=0))
def map_row() -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(collapsed=False,
repeat='view_id',
repeatDirection='h',
title=statistics_row_title,
type='row',
panels=[],
targets=[dict(refId='A',
datasource=datasource)],
gridPos=dict(h=1,
w=24,
x=0,
y=0))
def map_panels(dashboard: dict, database: Database) -> [dict]:
if map_statistics_row(dashboard) is None:
dashboard['panels'].append(map_row())
dashboard['panels'].append(map_overview_panel(database.id))
for view in database.views:
dashboard['panels'].append(map_statistics_panel(database.id, view))
return dashboard['panels']
def find(uid: str):
grafana = grafana_client.connect()
return grafana.dashboard.get_dashboard(uid)
def create(database_name: str, uid: str = '') -> dict:
grafana = grafana_client.connect()
dashboard = dict(uid=uid,
title=f'{database_name} Overview',
tags=['dbrepo'],
timezone='browser',
fiscalYearStartMonth=1,
panels=[])
dashboard['panels'] = []
payload = dict(folderUid='',
overwrite=False,
dashboard=dashboard)
dashboard = grafana.dashboard.update_dashboard(payload)
logging.info(f"Created dashboard with uid: {dashboard['uid']}")
return dashboard
def delete(uid: str) -> None:
grafana = grafana_client.connect()
grafana.dashboard.delete_dashboard(uid)
def update(database: Database) -> None:
grafana = grafana_client.connect()
dashboard = find(database.dashboard_uid)['dashboard']
# update metadata
if len(database.identifiers) > 0 and len(database.identifiers[0].titles) > 0:
dashboard['title'] = database.identifiers[0].titles[0].title
if len(database.identifiers) > 0 and len(database.identifiers[0].descriptions) > 0:
dashboard['description'] = database.identifiers[0].descriptions[0].description
dashboard['links'] = map_links(database)
dashboard['templating'] = map_templating(database)
# update panels
dashboard['panels'] = map_panels(dashboard, database)
payload = dict(folderUid='',
overwrite=True,
dashboard=dashboard)
response = grafana.dashboard.update_dashboard(payload)
logging.info(f"Updated dashboard with uid: {response['uid']}")
import os
datasource_uid = os.getenv('JSON_DATASOURCE_NAME', 'dbrepojson0')
statistics_row_title = '${view_id}'
def _get_start_index(dashboard: dict) -> int:
return [panel['title'] for panel in dashboard['panels']].index(statistics_row_title)
def get_panels(dashboard: dict) -> [dict]:
return []
def map_timeseries_panel(database_id: str) -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(title='${view_id}',
type='timeseries',
datasource=datasource,
targets=[dict(datasource=datasource,
format='table',
global_query_id='',
hide=False,
refId='A',
root_selector='',
source='url',
type='json',
url='/api/database/' + database_id + '/view/${view_id}/data',
url_options=dict(data='',
method='GET'))],
gridPos=dict(h=8,
w=12,
x=12,
y=8),
options=dict(legend=dict(displayMode='list',
placement='bottom',
showLegend=True),
tooltip=dict(mode='single',
sort='none')),
fieldConfig=dict(
defaults=dict(color=dict(mode='palette-classic'),
custom=dict(
axisBorderShow=False,
axisCenteredZero=False,
axisColorMode='text',
axisLabel='',
axisPlacement='auto',
barAlignment=0,
drawStyle='line',
fillOpacity=0,
gradientMode='none',
hideFrom=dict(legend=False,
tooltip=False,
viz=False),
insertNulls=False,
lineInterpolation='linear',
lineWidth=1,
pointSize=5,
scaleDistribution=dict(type='linear'),
showPoints='auto',
spanNulls=False,
stacking=dict(group='A',
mode='none'),
thresholdsStyle=dict(mode='absolute')))))
def map_number_panel(database_id: str, title: str, root_selector: str, y: int = 0) -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(title=title,
type='stat',
datasource=datasource,
targets=[dict(datasource=datasource,
columns=[],
filters=[],
format='table',
global_query_id='',
hide=False,
refId='A',
root_selector=root_selector,
source='url',
type='json',
url='/api/database/' + database_id + '/view/${view_id}/statistic',
url_options=dict(data='',
method='GET'))],
fieldConfig=dict(defaults=dict(mappings=[],
thresholds=dict(mode='absolute',
steps=[dict(color='blue',
value=None)]),
unit=''),
overrides=[]),
gridPos=dict(h=4,
w=6,
x=18,
y=y),
options=dict(colorMode='background',
graphMode='area',
justifyMode='auto',
orientation='auto',
reduceOptions=dict(calcs=[],
fields='/.*/',
values=True),
showPercentChange=False,
textMode='auto',
wideLayout=True))
def map_statistics_panel(database_id: str) -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(title='Statistics',
type='table',
gridPos=dict(h=8,
w=12,
x=0,
y=8),
datasource=datasource,
targets=[dict(datasource=datasource,
columns=[],
filters=[],
format='table',
global_query_id='',
hide=False,
refId='A',
root_selector='columns',
source='url',
type='json',
url='/api/database/' + database_id + '/view/${view_id}/statistic',
url_options=dict(data='',
method='GET'))],
options=dict(cellHeight="sm",
showHeader=True,
footer=dict(countRows=False,
fields="",
reducer=["sum"],
show=False)),
transformations=[dict(id="organize",
options=dict(excludeByName=dict(),
includeByName=dict(),
indexByName=dict(name=0,
val_min=1,
val_max=2,
mean=3,
median=4,
std_dev=5),
renameByName=dict(name="Name",
mean="Mean",
median="Median",
std_dev="std.dev",
val_min="Minimum",
val_max="Maximum")))],
fieldConfig=dict(defaults=dict(custom=dict(align="auto",
filterable="true",
cellOptions=dict(type="auto"),
inspect=False),
mappings=[],
thresholds=dict(mode="absolute",
steps=[dict(color="green",
value=None),
dict(color="red",
value=80)
])),
overrides=[]))
def map_overview_panel(database_id: str) -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(title='Datasource Preview',
type='table',
gridPos=dict(h=8,
w=18,
x=0,
y=4),
fieldConfig=dict(
defaults=dict(
color=dict(mode='palette-classic'),
custom=dict(axisBorderShow=False,
axisCenteredZero=False,
axisColorMode='text',
axisLabel='',
axisPlacement='auto',
barAlignment=0,
drawStyle='line',
fillOpacity=0,
gradientMode='none',
hideFrom=dict(
legend=False,
tooltip=False,
viz=False),
insertNulls=False,
lineInterpolation='linear',
lineWidth=1,
pointSize=5,
scaleDistribution=dict(
type='linear'),
showPoints='auto',
spanNulls=False,
stacking=dict(group='A',
mode='none'),
thresholdsStyle=dict(
mode='off'))),
overrides=[]),
options=dict(legend=dict(displayMode='list',
placement='bottom',
showLegend=True,
calcs=[]),
tooltip=dict(mode='single',
sort='none')),
targets=[dict(format='json',
columns=[],
datasource=datasource,
filters=[],
global_query_id='',
refId='A',
root_selector='',
source='url',
type='json',
url='/api/database/' + database_id + '/view/${view_id}/data',
url_options=dict(data='',
method='GET'))],
datasource=datasource)
def map_row() -> dict:
datasource = dict(uid=datasource_uid,
type='yesoreyeram-infinity-datasource')
return dict(collapsed=False,
repeat='view_id',
repeatDirection='h',
title=statistics_row_title,
type='row',
panels=[],
targets=[dict(refId='A',
datasource=datasource)],
gridPos=dict(h=1,
w=24,
x=0,
y=0))
def map_panels(dashboard: dict, database: Database) -> [dict]:
if get_statistics_row(dashboard) is None:
dashboard['panels'].append(map_row()) # repeating
dashboard['panels'].append(map_overview_panel(database.id)) # left top
dashboard['panels'].append(map_number_panel(database.id, 'Total Entries', 'rows', 0)) # right top
dashboard['panels'].append(map_number_panel(database.id, 'Variables', '$count(columns)', 4)) # right top
dashboard['panels'].append(map_statistics_panel(database.id)) # left
dashboard['panels'].append(map_timeseries_panel(database.id)) # middle
return dashboard['panels']
......@@ -16,7 +16,7 @@
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-data-service</artifactId>
<name>dbrepo-data-service</name>
<version>1.7.1</version>
<version>1.7.2</version>
<description>Service that manages the data</description>
......@@ -298,6 +298,12 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-metadata-service-entities</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
......
......@@ -6,12 +6,12 @@
<parent>
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-data-service</artifactId>
<version>1.7.1</version>
<version>1.7.2</version>
</parent>
<artifactId>dbrepo-data-service-querystore</artifactId>
<name>dbrepo-data-service-querystore</name>
<version>1.7.1</version>
<version>1.7.2</version>
<dependencies/>
......
......@@ -6,12 +6,12 @@
<parent>
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-data-service</artifactId>
<version>1.7.1</version>
<version>1.7.2</version>
</parent>
<artifactId>report</artifactId>
<name>dbrepo-data-service-report</name>
<version>1.7.1</version>
<version>1.7.2</version>
<description>
This module is only intended for the pipeline coverage report. See the detailed report in the
respective modules
......
......@@ -6,18 +6,18 @@
<parent>
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-data-service</artifactId>
<version>1.7.1</version>
<version>1.7.2</version>
</parent>
<artifactId>rest-service</artifactId>
<name>dbrepo-data-service-rest-service</name>
<version>1.7.1</version>
<version>1.7.2</version>
<dependencies>
<dependency>
<groupId>at.tuwien</groupId>
<artifactId>services</artifactId>
<version>1.7.1</version>
<version>1.7.2</version>
</dependency>
</dependencies>
......
......@@ -5,12 +5,12 @@ import at.tuwien.api.database.AccessTypeDto;
import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.internal.CreateDatabaseDto;
import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.api.user.UserDto;
import at.tuwien.api.user.internal.UpdateUserPasswordDto;
import at.tuwien.exception.*;
import at.tuwien.mapper.MetadataMapper;
import at.tuwien.service.AccessService;
import at.tuwien.service.ContainerService;
import at.tuwien.service.CacheService;
import at.tuwien.service.ContainerService;
import at.tuwien.service.DatabaseService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
......@@ -38,14 +38,16 @@ public class DatabaseEndpoint extends RestEndpoint {
private final CacheService cacheService;
private final AccessService accessService;
private final MetadataMapper metadataMapper;
private final DatabaseService databaseService;
private final ContainerService containerService;
@Autowired
public DatabaseEndpoint(CacheService cacheService, AccessService accessService, DatabaseService databaseService,
ContainerService containerService) {
ContainerService containerService, MetadataMapper metadataMapper) {
this.cacheService = cacheService;
this.accessService = accessService;
this.metadataMapper = metadataMapper;
this.databaseService = databaseService;
this.containerService = containerService;
}
......@@ -91,12 +93,7 @@ public class DatabaseEndpoint extends RestEndpoint {
try {
final DatabaseDto database = containerService.createDatabase(container, data);
containerService.createQueryStore(container, data.getInternalName());
final UserDto user = UserDto.builder()
.id(data.getUserId())
.username(data.getUsername())
.password(data.getPassword())
.build();
accessService.create(database, user, AccessTypeDto.WRITE_ALL);
accessService.create(database, metadataMapper.createDatabaseDtoToPrivilegedUserDto(data), AccessTypeDto.WRITE_ALL);
return ResponseEntity.status(HttpStatus.CREATED)
.body(database);
} catch (SQLException e) {
......
......@@ -279,7 +279,7 @@ public class SubsetEndpoint extends RestEndpoint {
log.debug("timestamp not set: default to {}", timestamp);
}
/* create */
final DatabaseDto database = cacheService.getDatabase(databaseId);
final DatabaseDto database = cacheService.getDatabase(databaseId, true);
if (!database.getIsSchemaPublic()) {
if (principal == null) {
log.error("Failed to create subset: no authentication found");
......
......@@ -24,6 +24,7 @@ import jakarta.validation.constraints.NotNull;
import lombok.extern.log4j.Log4j2;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.jooq.DSLContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
......@@ -44,6 +45,7 @@ import java.util.UUID;
@RequestMapping(path = "/api/database/{databaseId}/view")
public class ViewEndpoint extends RestEndpoint {
private final DSLContext context;
private final ViewService viewService;
private final CacheService cacheService;
private final MariaDbMapper mariaDbMapper;
......@@ -53,9 +55,11 @@ public class ViewEndpoint extends RestEndpoint {
private final EndpointValidator endpointValidator;
@Autowired
public ViewEndpoint(ViewService viewService, CacheService cacheService, MariaDbMapper mariaDbMapper,
SubsetService subsetService, StorageService storageService, DatabaseService databaseService,
public ViewEndpoint(DSLContext context, ViewService viewService, CacheService cacheService,
MariaDbMapper mariaDbMapper, SubsetService subsetService,
StorageService storageService, DatabaseService databaseService,
EndpointValidator endpointValidator) {
this.context = context;
this.viewService = viewService;
this.cacheService = cacheService;
this.mariaDbMapper = mariaDbMapper;
......@@ -155,11 +159,11 @@ public class ViewEndpoint extends RestEndpoint {
/* check */
endpointValidator.validateSubsetParams(data.getQuery());
/* create */
final DatabaseDto database = cacheService.getDatabase(databaseId);
final DatabaseDto database = cacheService.getDatabase(databaseId, true);
try {
return ResponseEntity.status(HttpStatus.CREATED)
.body(databaseService.createView(database, mariaDbMapper.nameToInternalName(data.getName()),
mariaDbMapper.subsetDtoToRawQuery(database, data.getQuery())));
mariaDbMapper.subsetDtoToRawQuery(context, database, data.getQuery())));
} catch (SQLException e) {
log.error("Failed to establish connection to database: {}", e.getMessage());
throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e);
......@@ -201,7 +205,7 @@ public class ViewEndpoint extends RestEndpoint {
ViewMalformedException, MetadataServiceException, DatabaseNotFoundException {
log.debug("endpoint delete view, databaseId={}, viewId={}", databaseId, viewId);
final ViewDto view = cacheService.getView(databaseId, viewId);
final DatabaseDto database = cacheService.getDatabase(databaseId);
final DatabaseDto database = cacheService.getDatabase(databaseId, true);
try {
viewService.delete(database, view);
return ResponseEntity.status(HttpStatus.ACCEPTED)
......
......@@ -6,7 +6,6 @@ import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.query.FilterDto;
import at.tuwien.api.database.query.FilterTypeDto;
import at.tuwien.api.database.query.SubsetDto;
import at.tuwien.config.QueryConfig;
import at.tuwien.endpoints.RestEndpoint;
import at.tuwien.exception.*;
import at.tuwien.service.CacheService;
......@@ -15,23 +14,17 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.security.Principal;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Log4j2
@Component
public class EndpointValidator extends RestEndpoint {
private final QueryConfig queryConfig;
private final CacheService credentialService;
@Autowired
public EndpointValidator(QueryConfig queryConfig, CacheService credentialService) {
this.queryConfig = queryConfig;
public EndpointValidator(CacheService credentialService) {
this.credentialService = credentialService;
}
......@@ -51,21 +44,6 @@ public class EndpointValidator extends RestEndpoint {
}
}
public void validateOnlyPrivateSchemaAccess(DatabaseDto database, Principal principal)
throws NotAllowedException, RemoteUnavailableException, MetadataServiceException {
validateOnlyPrivateSchemaAccess(database, principal, false);
}
public void validateOnlyPrivateSchemaAccess(DatabaseDto database, Principal principal,
boolean writeAccessOnly) throws NotAllowedException,
RemoteUnavailableException, MetadataServiceException {
if (database.getIsSchemaPublic()) {
log.trace("database schema with id {} is public: no access needed", database.getId());
return;
}
validateOnlyAccess(database, principal, writeAccessOnly);
}
public void validateSubsetParams(SubsetDto subset) throws QueryMalformedException {
if (subset.getFilter() != null) {
final List<FilterDto> filters = subset.getFilter();
......@@ -82,25 +60,6 @@ public class EndpointValidator extends RestEndpoint {
}
}
public void validateOnlyPrivateSchemaHasRole(DatabaseDto database, Principal principal, String role)
throws NotAllowedException {
if (database.getIsSchemaPublic()) {
log.trace("database with id {} has public schema: no access needed", database.getId());
return;
}
log.trace("database with id {} has private schema", database.getId());
if (principal == null) {
log.error("Access not allowed: no authorization provided");
throw new NotAllowedException("Access not allowed: no authorization provided");
}
log.trace("principal: {}", principal.getName());
if (!hasRole(principal, role)) {
log.error("Access not allowed: role {} missing", role);
throw new NotAllowedException("Access not allowed: role " + role + " missing");
}
log.trace("principal has role '{}': access granted", role);
}
public void validateOnlyAccess(DatabaseDto database, Principal principal, boolean writeAccessOnly)
throws NotAllowedException, RemoteUnavailableException, MetadataServiceException {
if (principal == null) {
......@@ -117,25 +76,6 @@ public class EndpointValidator extends RestEndpoint {
}
}
public void validateForbiddenStatements(String query) throws QueryNotSupportedException {
final List<String> words = new LinkedList<>();
Arrays.stream(queryConfig.getForbiddenKeywords())
.forEach(keyword -> {
final Pattern pattern = Pattern.compile("(" + keyword + ")");
final Matcher matcher = pattern.matcher(query);
final boolean found = matcher.find();
if (found) {
words.add(keyword);
log.debug("query contains keyword '{}' matching '{}'", keyword, matcher.group(1));
}
});
if (words.isEmpty()) {
return;
}
log.error("Query contains forbidden keyword(s): {}", words);
throw new QueryNotSupportedException("Query contains forbidden keyword(s): " + Arrays.toString(words.toArray()));
}
public void validateOnlyWriteOwnOrWriteAllAccess(AccessTypeDto access, UUID owner, UUID user) throws NotAllowedException {
if (access.equals(AccessTypeDto.READ)) {
log.error("Failed to create table data: no write access");
......
......@@ -63,7 +63,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
private DatabaseService databaseService;
@MockBean
private CacheService credentialService;
private CacheService cacheService;
@MockBean
private MetadataServiceGateway metadataServiceGateway;
......@@ -94,7 +94,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
MetadataServiceException {
/* mock */
when(credentialService.getAccess(DATABASE_3_ID, USER_3_ID))
when(cacheService.getAccess(DATABASE_3_ID, USER_3_ID))
.thenReturn(DATABASE_3_USER_1_READ_ACCESS_DTO);
when(subsetService.findAll(DATABASE_3_PRIVILEGED_DTO, null))
.thenReturn(List.of(QUERY_1_DTO, QUERY_2_DTO, QUERY_3_DTO, QUERY_4_DTO, QUERY_5_DTO, QUERY_6_DTO));
......@@ -120,7 +120,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
DatabaseNotFoundException, RemoteUnavailableException, MetadataServiceException {
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
doThrow(SQLException.class)
.when(subsetService)
......@@ -138,7 +138,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
RemoteUnavailableException, MetadataServiceException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(cacheService.getDatabase(DATABASE_1_ID))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
/* test */
......@@ -154,7 +154,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
QueryNotFoundException, MetadataServiceException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(cacheService.getDatabase(DATABASE_1_ID))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
when(subsetService.findById(DATABASE_1_PRIVILEGED_DTO, QUERY_1_ID))
.thenReturn(QUERY_1_DTO);
......@@ -170,7 +170,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
QueryNotFoundException, MetadataServiceException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(cacheService.getDatabase(DATABASE_1_ID))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
when(subsetService.findById(DATABASE_1_PRIVILEGED_DTO, QUERY_1_ID))
.thenReturn(QUERY_1_DTO);
......@@ -186,7 +186,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
QueryNotFoundException, MetadataServiceException {
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
when(subsetService.findById(DATABASE_3_PRIVILEGED_DTO, QUERY_5_ID))
.thenReturn(QUERY_5_DTO);
......@@ -202,7 +202,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
QueryNotFoundException, SQLException, MetadataServiceException, NotAllowedException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(cacheService.getDatabase(DATABASE_1_ID))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
when(subsetService.findById(DATABASE_1_PRIVILEGED_DTO, QUERY_1_ID))
.thenReturn(QUERY_5_DTO);
......@@ -218,7 +218,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
NotAllowedException, QueryNotFoundException, SQLException {
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
when(subsetService.findById(DATABASE_3_PRIVILEGED_DTO, QUERY_5_ID))
.thenReturn(QUERY_5_DTO);
......@@ -236,7 +236,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
when(subsetService.findById(DATABASE_3_PRIVILEGED_DTO, QUERY_5_ID))
.thenReturn(QUERY_5_DTO);
......@@ -256,7 +256,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* mock */
doThrow(DatabaseNotFoundException.class)
.when(credentialService)
.when(cacheService)
.getDatabase(DATABASE_3_ID);
/* test */
......@@ -271,7 +271,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
MetadataServiceException, SQLException, UserNotFoundException, QueryNotFoundException {
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
doThrow(SQLException.class)
.when(subsetService)
......@@ -294,7 +294,9 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID, true))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
when(subsetService.getData(any(DatabaseDto.class), anyString()))
.thenReturn(mock);
......@@ -324,7 +326,9 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID, true))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
when(subsetService.create(any(DatabaseDto.class), any(SubsetDto.class), any(Instant.class), eq(USER_1_ID)))
.thenReturn(QUERY_5_ID);
......@@ -352,8 +356,8 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* mock */
doThrow(DatabaseNotFoundException.class)
.when(credentialService)
.getDatabase(DATABASE_3_ID);
.when(cacheService)
.getDatabase(DATABASE_3_ID, true);
when(httpServletRequest.getMethod())
.thenReturn("POST");
......@@ -374,7 +378,9 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
when(credentialService.getDatabase(DATABASE_4_ID))
when(cacheService.getDatabase(DATABASE_4_ID, true))
.thenReturn(DATABASE_4_PRIVILEGED_DTO);
when(cacheService.getDatabase(DATABASE_4_ID))
.thenReturn(DATABASE_4_PRIVILEGED_DTO);
when(subsetService.findById(eq(DATABASE_4_PRIVILEGED_DTO), any(UUID.class)))
.thenReturn(QUERY_9_DTO);
......@@ -402,7 +408,9 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(cacheService.getDatabase(DATABASE_1_ID, true))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
when(cacheService.getDatabase(DATABASE_1_ID))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
when(subsetService.findById(any(DatabaseDto.class), any(UUID.class)))
.thenReturn(QUERY_1_DTO);
......@@ -429,7 +437,9 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
when(credentialService.getDatabase(DATABASE_2_ID))
when(cacheService.getDatabase(DATABASE_2_ID, true))
.thenReturn(DATABASE_2_PRIVILEGED_DTO);
when(cacheService.getDatabase(DATABASE_2_ID))
.thenReturn(DATABASE_2_PRIVILEGED_DTO);
when(subsetService.findById(eq(DATABASE_2_PRIVILEGED_DTO), any(UUID.class)))
.thenReturn(QUERY_8_DTO);
......@@ -454,7 +464,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
when(subsetService.findById(DATABASE_3_PRIVILEGED_DTO, QUERY_5_ID))
.thenReturn(QUERY_5_DTO);
......@@ -483,7 +493,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
ViewNotFoundException, ViewMalformedException, StorageUnavailableException, FormatNotAvailableException {
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
when(subsetService.findById(DATABASE_3_PRIVILEGED_DTO, QUERY_5_ID))
.thenReturn(QUERY_5_DTO);
......@@ -511,7 +521,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
final Dataset<Row> mock = sparkSession.emptyDataFrame();
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(cacheService.getDatabase(DATABASE_1_ID))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
when(subsetService.findById(DATABASE_1_PRIVILEGED_DTO, QUERY_1_ID))
.thenReturn(QUERY_1_DTO);
......@@ -539,7 +549,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
MetadataServiceException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(cacheService.getDatabase(DATABASE_1_ID))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
/* test */
......@@ -554,10 +564,10 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
NotAllowedException, MetadataServiceException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(cacheService.getDatabase(DATABASE_1_ID))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
doThrow(NotAllowedException.class)
.when(credentialService)
.when(cacheService)
.getAccess(DATABASE_1_ID, USER_1_ID);
/* test */
......@@ -575,7 +585,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
FormatNotAvailableException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(cacheService.getDatabase(DATABASE_1_ID))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
when(subsetService.findById(DATABASE_1_PRIVILEGED_DTO, QUERY_1_ID))
.thenReturn(QUERY_1_DTO);
......@@ -603,9 +613,9 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.build();
/* mock */
when(credentialService.getAccess(DATABASE_3_ID, USER_3_ID))
when(cacheService.getAccess(DATABASE_3_ID, USER_3_ID))
.thenReturn(DATABASE_3_USER_3_READ_ACCESS_DTO);
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
doNothing()
.when(subsetService)
......@@ -639,7 +649,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* mock */
doThrow(NotAllowedException.class)
.when(credentialService)
.when(cacheService)
.getAccess(DATABASE_3_ID, USER_3_ID);
/* test */
......@@ -657,10 +667,10 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.build();
/* mock */
when(credentialService.getAccess(DATABASE_3_ID, USER_3_ID))
when(cacheService.getAccess(DATABASE_3_ID, USER_3_ID))
.thenReturn(DATABASE_3_USER_3_READ_ACCESS_DTO);
doThrow(DatabaseNotFoundException.class)
.when(credentialService)
.when(cacheService)
.getDatabase(DATABASE_3_ID);
/* test */
......@@ -678,9 +688,9 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
.build();
/* mock */
when(credentialService.getDatabase(DATABASE_3_ID))
when(cacheService.getDatabase(DATABASE_3_ID))
.thenReturn(DATABASE_3_PRIVILEGED_DTO);
when(credentialService.getAccess(DATABASE_3_ID, USER_3_ID))
when(cacheService.getAccess(DATABASE_3_ID, USER_3_ID))
.thenReturn(DATABASE_3_USER_3_READ_ACCESS_DTO);
doThrow(SQLException.class)
.when(subsetService)
......@@ -698,11 +708,11 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest {
/* mock */
if (database != null) {
when(credentialService.getDatabase(databaseId))
when(cacheService.getDatabase(databaseId))
.thenReturn(database);
} else {
doThrow(DatabaseNotFoundException.class)
.when(credentialService)
.when(cacheService)
.getDatabase(databaseId);
}
......
......@@ -72,7 +72,7 @@ public class ViewEndpointUnitTest extends AbstractUnitTest {
ImageNotFoundException, QueryMalformedException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(credentialService.getDatabase(DATABASE_1_ID, true))
.thenReturn(DATABASE_1_DTO);
when(databaseService.createView(any(DatabaseDto.class), anyString(), anyString()))
.thenReturn(VIEW_1_DTO);
......@@ -88,7 +88,7 @@ public class ViewEndpointUnitTest extends AbstractUnitTest {
ViewMalformedException, MetadataServiceException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID))
when(credentialService.getDatabase(DATABASE_1_ID, true))
.thenReturn(DATABASE_1_DTO);
doThrow(SQLException.class)
.when(databaseService)
......@@ -125,7 +125,7 @@ public class ViewEndpointUnitTest extends AbstractUnitTest {
/* mock */
doThrow(DatabaseNotFoundException.class)
.when(credentialService)
.getDatabase(DATABASE_1_ID);
.getDatabase(DATABASE_1_ID, true);
/* test */
assertThrows(DatabaseNotFoundException.class, () -> {
......@@ -228,10 +228,10 @@ public class ViewEndpointUnitTest extends AbstractUnitTest {
MetadataServiceException, ViewNotFoundException, DatabaseNotFoundException {
/* mock */
when(credentialService.getDatabase(DATABASE_1_ID, true))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
when(credentialService.getView(DATABASE_1_ID, VIEW_1_ID))
.thenReturn(VIEW_1_DTO);
when(credentialService.getDatabase(DATABASE_1_ID))
.thenReturn(DATABASE_1_PRIVILEGED_DTO);
doThrow(SQLException.class)
.when(viewService)
.delete(DATABASE_1_PRIVILEGED_DTO, VIEW_1_DTO);
......
package at.tuwien.validation;
import at.tuwien.exception.PaginationException;
import at.tuwien.exception.QueryNotSupportedException;
import at.tuwien.test.AbstractUnitTest;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.Test;
......@@ -67,29 +66,4 @@ public class EndpointValidatorUnitTest extends AbstractUnitTest {
});
}
@Test
public void validateForbiddenStatements_succeeds() throws QueryNotSupportedException {
/* test */
endpointValidator.validateForbiddenStatements("SELECT country FROM some_table");
}
@Test
public void validateForbiddenStatements_fails() {
/* test */
assertThrows(QueryNotSupportedException.class, () -> {
endpointValidator.validateForbiddenStatements("SELECT COUNT(id) FROM some_table");
});
}
@Test
public void validateForbiddenStatements_lowercase_fails() {
/* test */
assertThrows(QueryNotSupportedException.class, () -> {
endpointValidator.validateForbiddenStatements("SELECT COUNT(id) FROM some_table");
});
}
}
......@@ -6,18 +6,18 @@
<parent>
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-data-service</artifactId>
<version>1.7.1</version>
<version>1.7.2</version>
</parent>
<artifactId>services</artifactId>
<name>dbrepo-data-service-services</name>
<version>1.7.1</version>
<version>1.7.2</version>
<dependencies>
<dependency>
<groupId>at.tuwien</groupId>
<artifactId>dbrepo-data-service-querystore</artifactId>
<version>1.7.1</version>
<version>1.7.2</version>
</dependency>
</dependencies>
......
......@@ -2,15 +2,29 @@ package at.tuwien.config;
import lombok.Getter;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Value;
import org.jooq.DSLContext;
import org.jooq.SQLDialect;
import org.jooq.conf.Settings;
import org.jooq.conf.StatementType;
import org.jooq.impl.DefaultConfiguration;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import static org.jooq.impl.DSL.using;
@Log4j2
@Getter
@Configuration
public class QueryConfig {
@Value("${dbrepo.sql.forbidden}")
private String[] forbiddenKeywords;
@Bean
public DSLContext context() {
final DefaultConfiguration configuration = new DefaultConfiguration();
final Settings settings = new Settings();
settings.setStatementType(StatementType.STATIC_STATEMENT);
configuration.setSettings(settings);
configuration.set(SQLDialect.MARIADB);
return using(configuration);
}
}
......@@ -39,7 +39,7 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.jooq.impl.DSL.field;
import static org.jooq.impl.DSL.select;
import static org.jooq.impl.DSL.name;
@Mapper(componentModel = "spring", uses = {MetadataMapper.class, DataMapper.class})
public interface MariaDbMapper {
......@@ -79,7 +79,6 @@ public interface MariaDbMapper {
.append("`@`%` IDENTIFIED BY PASSWORD '")
.append(password)
.append("';");
log.trace("mapped create user statement: {}", statement);
return statement.toString();
}
......@@ -107,7 +106,7 @@ public interface MariaDbMapper {
.append("` TO `")
.append(username)
.append("`@`%`;");
log.trace("mapped revoke privileges statement: {}", statement);
log.trace("mapped grant privileges statement: {}", statement);
return statement.toString();
}
......@@ -181,7 +180,7 @@ public interface MariaDbMapper {
}
default String databaseTableSelectRawQuery() {
final String statement = "SELECT t.`TABLE_NAME`, t.`TABLE_TYPE`, t.`TABLE_ROWS`, t.`AVG_ROW_LENGTH`, t.`DATA_LENGTH`, t.`MAX_DATA_LENGTH`, COALESCE(t.`CREATE_TIME`, NOW()) as `CREATE_TIME`, t.`UPDATE_TIME`, v.`VIEW_DEFINITION`, t.`TABLE_COMMENT` FROM information_schema.TABLES t LEFT JOIN information_schema.VIEWS v ON t.`TABLE_NAME` = v.`TABLE_NAME` WHERE t.`TABLE_SCHEMA` = ? AND t.`TABLE_TYPE` = 'SYSTEM VERSIONED' AND t.`TABLE_NAME` != 'qs_queries' AND t.`TABLE_NAME` = ?";
final String statement = "SELECT t.`TABLE_NAME`, t.`TABLE_TYPE`, t.`TABLE_ROWS`, t.`AVG_ROW_LENGTH`, t.`DATA_LENGTH`, t.`MAX_DATA_LENGTH`, COALESCE(t.`CREATE_TIME`, NOW()) as `CREATE_TIME`, t.`UPDATE_TIME`, v.`VIEW_DEFINITION`, t.`TABLE_COMMENT` FROM information_schema.TABLES t LEFT JOIN information_schema.VIEWS v ON t.`TABLE_NAME` = v.`TABLE_NAME` WHERE t.`TABLE_SCHEMA` = ? AND t.`TABLE_TYPE` IN ('SYSTEM VERSIONED', 'VIEW') AND t.`TABLE_NAME` != 'qs_queries' AND t.`TABLE_NAME` = ?";
log.trace("mapped select table statement: {}", statement);
return statement;
}
......@@ -846,33 +845,33 @@ public interface MariaDbMapper {
switch (operator) {
case "=":
case "<=>":
return field(column.getInternalName()).eq(data.getValue());
return field(name(column.getInternalName())).eq(data.getValue());
case "<":
return field(column.getInternalName()).lt(data.getValue());
return field(name(column.getInternalName())).lt(data.getValue());
case "<=":
return field(column.getInternalName()).le(data.getValue());
return field(name(column.getInternalName())).le(data.getValue());
case ">":
return field(column.getInternalName()).gt(data.getValue());
return field(name(column.getInternalName())).gt(data.getValue());
case ">=":
return field(column.getInternalName()).ge(data.getValue());
return field(name(column.getInternalName())).ge(data.getValue());
case "!=":
return field(column.getInternalName()).ne(data.getValue());
return field(name(column.getInternalName())).ne(data.getValue());
case "LIKE":
return field(column.getInternalName()).like(data.getValue());
return field(name(column.getInternalName())).like(data.getValue());
case "NOT LIKE":
return field(column.getInternalName()).notLike(data.getValue());
return field(name(column.getInternalName())).notLike(data.getValue());
case "IN":
return field(column.getInternalName()).in(data.getValue());
return field(name(column.getInternalName())).in(data.getValue());
case "NOT IN":
return field(column.getInternalName()).notIn(data.getValue());
return field(name(column.getInternalName())).notIn(data.getValue());
case "IS NOT NULL":
return field(column.getInternalName()).isNotNull();
return field(name(column.getInternalName())).isNotNull();
case "IS NULL":
return field(column.getInternalName()).isNull();
return field(name(column.getInternalName())).isNull();
case "REGEXP":
return field(column.getInternalName()).likeRegex(data.getValue());
return field(name(column.getInternalName())).likeRegex(data.getValue());
case "NOT REGEXP":
return field(column.getInternalName()).notLikeRegex(data.getValue());
return field(name(column.getInternalName())).notLikeRegex(data.getValue());
}
log.error("Failed to map operator: {}", operator);
throw new IllegalArgumentException("Failed to map operator: " + operator);
......@@ -884,26 +883,27 @@ public interface MariaDbMapper {
for (OrderDto order : data.getOrder()) {
final ColumnDto column = columnIdToColumnDto(database, order.getColumnId());
if (order.getDirection() == null) {
sort.add(field(column.getInternalName()));
sort.add(field(name(column.getInternalName())));
continue;
}
switch (order.getDirection()) {
case ASC -> sort.add(field(column.getInternalName()).asc());
case DESC -> sort.add(field(column.getInternalName()).desc());
case ASC -> sort.add(field(name(column.getInternalName())).asc());
case DESC -> sort.add(field(name(column.getInternalName())).desc());
}
}
return step.orderBy(sort);
}
default String subsetDtoToRawQuery(DatabaseDto database, SubsetDto data) throws TableNotFoundException, ImageNotFoundException {
default String subsetDtoToRawQuery(DSLContext context, DatabaseDto database, SubsetDto data)
throws TableNotFoundException, ImageNotFoundException {
final TableDto table = tableIdToTableDto(database, data.getTableId());
final List<Field<Object>> columns = table.getColumns()
.stream()
.filter(c -> data.getColumns().contains(c.getId()))
.map(c -> field(c.getInternalName()))
.map(c -> field(name(c.getInternalName())))
.toList();
final SelectJoinStep<Record> query = select(columns)
.from(table.getInternalName());
final SelectJoinStep<Record> query = context.select(columns)
.from(name(table.getInternalName()));
final SelectConditionStep<Record> where = subsetDtoToSelectConditions(query, database, data);
final String sql;
if (data.getOrder() == null) {
......@@ -966,7 +966,8 @@ public interface MariaDbMapper {
.findFirst();
if (optional.isEmpty()) {
log.error("Failed to find table with id: {}", tableId);
throw new TableNotFoundException("Failed to find table");
log.trace("known table ids: {}", database.getTables().stream().map(TableDto::getId).collect(Collectors.toList()));
throw new TableNotFoundException("Failed to find table id: " + tableId);
}
return optional.get();
}
......
......@@ -6,6 +6,8 @@ import at.tuwien.api.database.DatabaseBriefDto;
import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.ViewColumnDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.internal.CreateDatabaseDto;
import at.tuwien.api.database.query.QueryDto;
import at.tuwien.api.database.table.TableBriefDto;
import at.tuwien.api.database.table.TableDto;
import at.tuwien.api.database.table.columns.ColumnDto;
......@@ -28,6 +30,13 @@ public interface MetadataMapper {
ContainerDto containerDtoToContainerDto(ContainerDto data);
@Mappings({
@Mapping(target = "id", source = "userId"),
@Mapping(target = "username", source = "privilegedUsername"),
@Mapping(target = "password", source = "privilegedPassword"),
})
UserDto createDatabaseDtoToPrivilegedUserDto(CreateDatabaseDto data);
DatabaseBriefDto databaseDtoToDatabaseBriefDto(DatabaseDto data);
ColumnDto viewColumnDtoToColumnDto(ViewColumnDto data);
......
......@@ -13,6 +13,9 @@ import java.util.UUID;
public interface CacheService {
DatabaseDto getDatabase(UUID id, Boolean forceReload) throws DatabaseNotFoundException, RemoteUnavailableException,
MetadataServiceException;
/**
* Gets credentials for a database with given id either from the cache (if not expired) or retrieves them from the
* Metadata Service.
......
......@@ -5,7 +5,6 @@ import at.tuwien.api.database.DatabaseAccessDto;
import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.table.TableDto;
import at.tuwien.api.keycloak.TokenDto;
import at.tuwien.api.user.UserDto;
import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
......@@ -44,19 +43,27 @@ public class CacheServiceImpl implements CacheService {
}
@Override
public DatabaseDto getDatabase(UUID id) throws DatabaseNotFoundException, RemoteUnavailableException,
public DatabaseDto getDatabase(UUID id, Boolean forceReload) throws DatabaseNotFoundException, RemoteUnavailableException,
MetadataServiceException {
if (!forceReload) {
final DatabaseDto cacheDatabase = databaseCache.getIfPresent(id);
if (cacheDatabase != null) {
log.trace("found database with id {} in cache", id);
return cacheDatabase;
}
log.debug("database with id {} not it cache (anymore): reload from metadata service", id);
}
final DatabaseDto database = gateway.getDatabaseById(id);
databaseCache.put(id, database);
return database;
}
@Override
public DatabaseDto getDatabase(UUID id) throws DatabaseNotFoundException, RemoteUnavailableException,
MetadataServiceException {
return getDatabase(id, false);
}
@Override
public TableDto getTable(UUID databaseId, UUID tableId) throws RemoteUnavailableException,
MetadataServiceException, TableNotFoundException {
......
......@@ -156,7 +156,8 @@ public class StorageServiceS3Impl implements StorageService {
log.debug("read dataset from s3 path: {} using header: {}", path, withHeader);
Dataset<Row> dataset;
try {
log.trace("spark read conf: header={}, delimiter={}", withHeader, delimiter);
final String logDelimiter = delimiter.equals("\t") ? "[tab]" : delimiter;
log.trace("spark read conf: header={}, delimiter={}", withHeader, logDelimiter);
dataset = sparkSession.read()
.option("delimiter", delimiter)
.option("header", withHeader)
......