Skip to content
Snippets Groups Projects
Verified Commit 4045204d authored by Martin Weise's avatar Martin Weise
Browse files

Needed to fix the foreign key detection (again)

parent ba2f1021
No related branches found
No related tags found
4 merge requests!296Dev,!293Dev,!289Dev,!287Dev
Showing
with 685 additions and 699 deletions
......@@ -2,7 +2,7 @@
APP_VERSION ?= 1.4.4
CHART_VERSION ?= 1.4.4
REPOSITORY_URL ?= docker.io/dbrepo
REPOSITORY_URL ?= registry.datalab.tuwien.ac.at/dbrepo
.PHONY: all
all: help
......
......@@ -8,7 +8,6 @@ import at.tuwien.api.user.UserDto;
import at.tuwien.exception.*;
import at.tuwien.gateway.MetadataServiceGateway;
import at.tuwien.service.AccessService;
import io.micrometer.observation.annotation.Observed;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
......
......@@ -5,7 +5,6 @@ import at.tuwien.api.database.DatabaseAccessDto;
import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.query.ImportCsvDto;
import at.tuwien.api.database.query.QueryDto;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.api.database.table.*;
import at.tuwien.api.database.table.internal.PrivilegedTableDto;
......
......@@ -10,6 +10,7 @@ import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.internal.PrivilegedViewDto;
import at.tuwien.api.database.table.TableBriefDto;
import at.tuwien.api.database.table.TableDto;
import at.tuwien.api.database.table.columns.ColumnBriefDto;
import at.tuwien.api.database.table.columns.ColumnDto;
import at.tuwien.api.database.table.internal.PrivilegedTableDto;
import at.tuwien.api.user.PrivilegedUserDto;
......@@ -33,9 +34,6 @@ public interface MetadataMapper {
ViewColumnDto columnDtoToViewColumnDto(ColumnDto data);
/* keep */
TableBriefDto tableDtoToTableBriefDto(TableDto data);
@Mappings({
@Mapping(target = "database", expression = "java(PrivilegedDatabaseDto.builder().container(PrivilegedContainerDto.builder().image(new ImageDto()).build()).build())")
})
......@@ -47,4 +45,9 @@ public interface MetadataMapper {
PrivilegedUserDto userDtoToPrivilegedUserDto(UserDto data);
@Mappings({
@Mapping(target = "databaseId", source = "tdbid")
})
TableBriefDto tableDtoToTableBriefDto(TableDto data);
}
package at.tuwien.service.impl;
import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.ViewColumnDto;
import at.tuwien.api.database.ViewDto;
import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
import at.tuwien.api.database.table.TableDto;
import at.tuwien.api.database.table.columns.ColumnDto;
import at.tuwien.api.database.table.constraints.unique.UniqueDto;
import at.tuwien.config.QueryConfig;
import at.tuwien.exception.TableNotFoundException;
import at.tuwien.exception.ViewMalformedException;
import at.tuwien.exception.ViewNotFoundException;
import at.tuwien.exception.ViewSchemaException;
import at.tuwien.mapper.DataMapper;
import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.mapper.MetadataMapper;
import at.tuwien.service.SchemaService;
......@@ -25,20 +22,20 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
@Log4j2
@Service
public class SchemaServiceMariaDbImpl extends HibernateConnector implements SchemaService {
private final DataMapper dataMapper;
private final QueryConfig queryConfig;
private final MariaDbMapper mariaDbMapper;
private final MetadataMapper metadataMapper;
@Autowired
public SchemaServiceMariaDbImpl(QueryConfig queryConfig, MariaDbMapper mariaDbMapper,
public SchemaServiceMariaDbImpl(DataMapper dataMapper, QueryConfig queryConfig, MariaDbMapper mariaDbMapper,
MetadataMapper metadataMapper) {
this.dataMapper = dataMapper;
this.queryConfig = queryConfig;
this.mariaDbMapper = mariaDbMapper;
this.metadataMapper = metadataMapper;
......@@ -56,7 +53,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
statement1.setString(1, database.getInternalName());
statement1.setString(2, tableName);
log.trace("1={}, 2={}", database.getInternalName(), tableName);
TableDto table = mariaDbMapper.schemaResultSetToTable(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), statement1.executeQuery());
TableDto table = dataMapper.schemaResultSetToTable(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), statement1.executeQuery());
/* obtain columns metadata */
final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery());
statement2.setString(1, database.getInternalName());
......@@ -64,7 +61,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
log.trace("1={}, 2={}", database.getInternalName(), tableName);
final ResultSet resultSet2 = statement2.executeQuery();
while (resultSet2.next()) {
table = mariaDbMapper.resultSetToTable(resultSet2, table, queryConfig);
table = dataMapper.resultSetToTable(resultSet2, table, queryConfig);
}
/* obtain check constraints metadata */
final PreparedStatement statement3 = connection.prepareStatement(mariaDbMapper.columnsCheckConstraintSelectRawQuery());
......@@ -86,7 +83,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
log.trace("1={}, 2={}", database.getInternalName(), tableName);
final ResultSet resultSet4 = statement4.executeQuery();
while (resultSet4.next()) {
table = mariaDbMapper.resultSetToConstraint(resultSet4, table);
table = dataMapper.resultSetToConstraint(resultSet4, table);
for (UniqueDto uk : table.getConstraints().getUniques()) {
uk.setTable(metadataMapper.tableDtoToTableBriefDto(table));
final TableDto tmpTable = table;
......@@ -133,7 +130,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
if (!resultSet1.next()) {
throw new ViewNotFoundException("Failed to find view in the information schema");
}
ViewDto view = mariaDbMapper.schemaResultSetToView(metadataMapper.privilegedDatabaseDtoToDatabaseDto(privilegedDatabase), resultSet1);
ViewDto view = dataMapper.schemaResultSetToView(metadataMapper.privilegedDatabaseDtoToDatabaseDto(privilegedDatabase), resultSet1);
view.setDatabase(database);
view.setVdbid(database.getId());
view.setCreator(database.getCreator());
......@@ -148,7 +145,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche
.columns(new LinkedList<>())
.build();
while (resultSet2.next()) {
tmp = mariaDbMapper.resultSetToTable(resultSet2, tmp, queryConfig);
tmp = dataMapper.resultSetToTable(resultSet2, tmp, queryConfig);
}
view.setColumns(tmp.getColumns()
.stream()
......
......@@ -14,6 +14,7 @@ import at.tuwien.config.S3Config;
import at.tuwien.exception.*;
import at.tuwien.gateway.DataDatabaseSidecarGateway;
import at.tuwien.gateway.MetadataServiceGateway;
import at.tuwien.mapper.DataMapper;
import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.mapper.MetadataMapper;
import at.tuwien.service.SubsetService;
......@@ -35,6 +36,7 @@ import java.util.UUID;
public class SubsetServiceMariaDbImpl extends HibernateConnector implements SubsetService {
private final S3Config s3Config;
private final DataMapper dataMapper;
private final MariaDbMapper mariaDbMapper;
private final MetadataMapper metadataMapper;
private final StorageService storageService;
......@@ -42,10 +44,12 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
private final DataDatabaseSidecarGateway dataDatabaseSidecarGateway;
@Autowired
public SubsetServiceMariaDbImpl(S3Config s3Config, MariaDbMapper mariaDbMapper, MetadataMapper metadataMapper,
StorageService storageService, MetadataServiceGateway metadataServiceGateway,
public SubsetServiceMariaDbImpl(S3Config s3Config, DataMapper dataMapper, MariaDbMapper mariaDbMapper,
MetadataMapper metadataMapper, StorageService storageService,
MetadataServiceGateway metadataServiceGateway,
DataDatabaseSidecarGateway dataDatabaseSidecarGateway) {
this.s3Config = s3Config;
this.dataMapper = dataMapper;
this.mariaDbMapper = mariaDbMapper;
this.metadataMapper = metadataMapper;
this.storageService = storageService;
......@@ -97,7 +101,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
SQLException {
final List<ColumnDto> columns;
try {
columns = mariaDbMapper.parseColumns(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), query.getQuery());
columns = dataMapper.parseColumns(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), query.getQuery());
} catch (JSQLParserException e) {
log.error("Failed to map/parse columns: {}", e.getMessage());
throw new TableMalformedException("Failed to map/parse columns: " + e.getMessage(), e);
......@@ -129,7 +133,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
final ResultSet resultSet = statement.executeQuery();
final List<QueryDto> queries = new LinkedList<>();
while (resultSet.next()) {
final QueryDto query = mariaDbMapper.resultSetToQueryDto(resultSet);
final QueryDto query = dataMapper.resultSetToQueryDto(resultSet);
query.setIdentifiers(identifiers.stream()
.filter(i -> i.getType().equals(IdentifierTypeDto.SUBSET))
.filter(i -> i.getQueryId().equals(query.getId()))
......@@ -176,7 +180,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
try {
final PreparedStatement preparedStatement = connection.prepareStatement(statement);
final ResultSet resultSet = preparedStatement.executeQuery();
return mariaDbMapper.resultListToQueryResultDto(columns, resultSet);
return dataMapper.resultListToQueryResultDto(columns, resultSet);
} catch (SQLException e) {
log.error("Failed to execute and map time-versioned query: {}", e.getMessage());
throw new TableMalformedException("Failed to execute and map time-versioned query: " + e.getMessage(), e);
......@@ -214,7 +218,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs
if (!resultSet.next()) {
throw new QueryNotFoundException("Failed to find query");
}
final QueryDto query = mariaDbMapper.resultSetToQueryDto(resultSet);
final QueryDto query = dataMapper.resultSetToQueryDto(resultSet);
query.setIdentifiers(metadataServiceGateway.getIdentifiers(database.getId(), queryId));
final UserDto creator = metadataServiceGateway.getUserById(query.getCreatedBy());
log.debug("retrieved creator from metadata service: creator.id={}, creator.username={}", creator.getId(), creator.getUsername());
......
......@@ -13,6 +13,7 @@ import at.tuwien.api.database.table.internal.TableCreateDto;
import at.tuwien.config.S3Config;
import at.tuwien.exception.*;
import at.tuwien.gateway.DataDatabaseSidecarGateway;
import at.tuwien.mapper.DataMapper;
import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.service.SchemaService;
import at.tuwien.service.StorageService;
......@@ -33,16 +34,18 @@ import java.util.*;
public class TableServiceMariaDbImpl extends HibernateConnector implements TableService {
private final S3Config s3Config;
private final DataMapper dataMapper;
private final MariaDbMapper mariaDbMapper;
private final SchemaService schemaService;
private final StorageService storageService;
private final DataDatabaseSidecarGateway dataDatabaseSidecarGateway;
@Autowired
public TableServiceMariaDbImpl(S3Config s3Config, MariaDbMapper mariaDbMapper, SchemaService schemaService,
StorageService storageService,
public TableServiceMariaDbImpl(S3Config s3Config, DataMapper dataMapper, MariaDbMapper mariaDbMapper,
SchemaService schemaService, StorageService storageService,
DataDatabaseSidecarGateway dataDatabaseSidecarGateway) {
this.s3Config = s3Config;
this.dataMapper = dataMapper;
this.mariaDbMapper = mariaDbMapper;
this.schemaService = schemaService;
this.storageService = storageService;
......@@ -91,7 +94,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
/* obtain statistic */
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.tableColumnStatisticsSelectRawQuery(table.getColumns(), table.getInternalName()))
.executeQuery();
statistic = mariaDbMapper.resultSetToTableStatistic(resultSet);
statistic = dataMapper.resultSetToTableStatistic(resultSet);
statistic.setRows(getCount(table, null));
} catch (SQLException e) {
connection.rollback();
......@@ -181,7 +184,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
timestamp, size, page))
.executeQuery();
connection.commit();
queryResult = mariaDbMapper.resultListToQueryResultDto(table.getColumns(), resultSet);
queryResult = dataMapper.resultListToQueryResultDto(table.getColumns(), resultSet);
} catch (SQLException e) {
connection.rollback();
log.error("Failed to find data from table {}.{}: {}", table.getDatabase().getInternalName(), table.getInternalName(), e.getMessage());
......@@ -205,7 +208,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectHistoryRawQuery(
table.getDatabase().getInternalName(), table.getInternalName(), size))
.executeQuery();
history = mariaDbMapper.resultSetToTableHistory(resultSet);
history = dataMapper.resultSetToTableHistory(resultSet);
connection.commit();
} catch (SQLException e) {
connection.rollback();
......
......@@ -11,6 +11,7 @@ import at.tuwien.config.QueryConfig;
import at.tuwien.config.S3Config;
import at.tuwien.exception.*;
import at.tuwien.gateway.DataDatabaseSidecarGateway;
import at.tuwien.mapper.DataMapper;
import at.tuwien.mapper.MariaDbMapper;
import at.tuwien.mapper.MetadataMapper;
import at.tuwien.service.SchemaService;
......@@ -37,6 +38,7 @@ import java.util.List;
public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewService {
private final S3Config s3Config;
private final DataMapper dataMapper;
private final QueryConfig queryConfig;
private final MariaDbMapper mariaDbMapper;
private final SchemaService schemaService;
......@@ -45,11 +47,12 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
private final DataDatabaseSidecarGateway dataDatabaseSidecarGateway;
@Autowired
public ViewServiceMariaDbImpl(S3Config s3Config, QueryConfig queryConfig, MariaDbMapper mariaDbMapper,
SchemaService schemaService, StorageService storageService,
MetadataMapper metadataMapper,
public ViewServiceMariaDbImpl(S3Config s3Config, DataMapper dataMapper, QueryConfig queryConfig,
MariaDbMapper mariaDbMapper, SchemaService schemaService,
StorageService storageService, MetadataMapper metadataMapper,
DataDatabaseSidecarGateway dataDatabaseSidecarGateway) {
this.s3Config = s3Config;
this.dataMapper = dataMapper;
this.queryConfig = queryConfig;
this.mariaDbMapper = mariaDbMapper;
this.schemaService = schemaService;
......@@ -122,7 +125,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
statement2.setString(2, view.getInternalName());
final ResultSet resultSet2 = statement2.executeQuery();
while (resultSet2.next()) {
view = mariaDbMapper.resultSetToTable(resultSet2, view, queryConfig);
view = dataMapper.resultSetToTable(resultSet2, view, queryConfig);
}
connection.commit();
} catch (SQLException e) {
......@@ -152,7 +155,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe
mariaDbMapper.selectDatasetRawQuery(view.getDatabase().getInternalName(),
view.getInternalName(), mappedColumns, timestamp, size, page))
.executeQuery();
queryResult = mariaDbMapper.resultListToQueryResultDto(mappedColumns, resultSet);
queryResult = dataMapper.resultListToQueryResultDto(mappedColumns, resultSet);
queryResult.setId(view.getId());
connection.commit();
} catch (SQLException e) {
......
......@@ -178,7 +178,7 @@ datadb:
protocol: TCP
sidecars:
- name: sidecar
image: s210.dl.hpc.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.4
imagePullPolicy: Always
securityContext:
runAsUser: 1001
......@@ -338,6 +338,8 @@ uploadservice:
tag: v1.12
securityContext:
allowPrivilegeEscalation: false
runAsUser: 1000
runAsGroup: 1000
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
......@@ -446,7 +448,7 @@ brokerservice:
analyseservice:
enabled: true
image:
name: s210.dl.hpc.tuwien.ac.at/dbrepo/analyse-service:1.4.4
name: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.4
pullPolicy: Always
debug: false
endpoint: http://analyse-service
......@@ -478,7 +480,7 @@ analyseservice:
metadataservice:
enabled: true
image:
name: s210.dl.hpc.tuwien.ac.at/dbrepo/metadata-service:1.4.4
name: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.4
pullPolicy: Always
debug: false
endpoint: http://metadata-service
......@@ -529,7 +531,7 @@ dataservice:
enabled: true
endpoint: http://data-service
image:
name: s210.dl.hpc.tuwien.ac.at/dbrepo/data-service:1.4.4
name: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.4
pullPolicy: Always
debug: false
grant:
......@@ -565,12 +567,12 @@ searchservice:
enabled: true
endpoint: http://search-service
image:
name: s210.dl.hpc.tuwien.ac.at/dbrepo/search-service:1.4.4
name: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.4
pullPolicy: Always
debug: false
init:
image:
name: s210.dl.hpc.tuwien.ac.at/dbrepo/search-service-init:1.4.4
name: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.4
pullPolicy: Always
replicaCount: 2
......@@ -617,7 +619,7 @@ storageservice:
username: seaweedfsadmin
password: seaweedfsadmin
init:
image: s210.dl.hpc.tuwien.ac.at/dbrepo/storage-service-init:1.4.4
image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.4.4
pullPolicy: Always
## @section User Interface
......@@ -646,7 +648,7 @@ storageservice:
ui:
enabled: true
image:
name: s210.dl.hpc.tuwien.ac.at/dbrepo/ui:1.4.4
name: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.4
pullPolicy: Always
debug: false
public:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment