diff --git a/Makefile b/Makefile index ea9a9d0ce3f29abf801f4a7769ad788259d33629..d3438ab7d5cfa78cfc7a5b829c37b917c14a42a9 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ APP_VERSION ?= 1.4.4 CHART_VERSION ?= 1.4.4 -REPOSITORY_URL ?= docker.io/dbrepo +REPOSITORY_URL ?= registry.datalab.tuwien.ac.at/dbrepo .PHONY: all all: help diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java index 95332db436c0c2ee56ebb3c91b090fe2b4144e8a..4966e008424aa6e1f290c70b29e97d4b55e72273 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/AccessEndpoint.java @@ -8,7 +8,6 @@ import at.tuwien.api.user.UserDto; import at.tuwien.exception.*; import at.tuwien.gateway.MetadataServiceGateway; import at.tuwien.service.AccessService; -import io.micrometer.observation.annotation.Observed; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java index a0200609a6001638615c7726279483f563585f97..e8a447c24f4ec1b077a67239bf2b94c75021bb5d 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java @@ -5,7 +5,6 @@ import at.tuwien.api.database.DatabaseAccessDto; import at.tuwien.api.database.DatabaseDto; import at.tuwien.api.database.internal.PrivilegedDatabaseDto; import at.tuwien.api.database.query.ImportCsvDto; -import at.tuwien.api.database.query.QueryDto; import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.table.*; import at.tuwien.api.database.table.internal.PrivilegedTableDto; diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java index 1516d698bdf0fbe1d702abfb32ff23b9303ac98c..163ec5940fb36d2a228f61a745773cbdea9f0a8f 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java @@ -1,23 +1,80 @@ package at.tuwien.mapper; +import at.tuwien.api.container.image.ImageDateDto; +import at.tuwien.api.database.DatabaseDto; +import at.tuwien.api.database.ViewColumnDto; +import at.tuwien.api.database.ViewDto; +import at.tuwien.api.database.query.QueryDto; +import at.tuwien.api.database.query.QueryResultDto; +import at.tuwien.api.database.table.TableBriefDto; import at.tuwien.api.database.table.TableDto; +import at.tuwien.api.database.table.TableHistoryDto; +import at.tuwien.api.database.table.TableStatisticDto; +import at.tuwien.api.database.table.columns.ColumnBriefDto; import at.tuwien.api.database.table.columns.ColumnDto; +import at.tuwien.api.database.table.columns.ColumnStatisticDto; import at.tuwien.api.database.table.columns.ColumnTypeDto; +import at.tuwien.api.database.table.constraints.ConstraintsDto; +import at.tuwien.api.database.table.constraints.foreign.ForeignKeyBriefDto; +import at.tuwien.api.database.table.constraints.foreign.ForeignKeyDto; +import at.tuwien.api.database.table.constraints.foreign.ForeignKeyReferenceDto; +import at.tuwien.api.database.table.constraints.foreign.ReferenceTypeDto; +import at.tuwien.api.database.table.constraints.primary.PrimaryKeyDto; +import at.tuwien.api.database.table.constraints.unique.UniqueDto; +import at.tuwien.config.QueryConfig; +import at.tuwien.exception.QueryNotFoundException; +import at.tuwien.exception.TableNotFoundException; +import com.github.dockerjava.zerodep.shaded.org.apache.commons.codec.binary.Hex; +import com.google.common.hash.Hashing; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserManager; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.*; +import org.jetbrains.annotations.NotNull; import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.Mappings; import org.testcontainers.shaded.org.apache.commons.io.FileUtils; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.sql.*; -import java.util.Map; +import java.sql.Date; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; @Mapper(componentModel = "spring") public interface DataMapper { org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(DataMapper.class); + DateTimeFormatter mariaDbFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss[.SSSSSS]") + .withZone(ZoneId.of("UTC")); + + /* redundant */ + ColumnBriefDto columnDtoToColumnBriefDto(ColumnDto data); + + /* redundant */ + @Mappings({ + @Mapping(target = "databaseId", source = "tdbid") + }) + TableBriefDto tableDtoToTableBriefDto(TableDto data); + + /* redundant */ + ColumnDto viewColumnDtoToColumnDto(ViewColumnDto data); + + ForeignKeyBriefDto foreignKeyDtoToForeignKeyBriefDto(ForeignKeyDto data); + default String rabbitMqTupleToInsertOrUpdateQuery(TableDto table, Map<String, Object> data) { /* parameterized query for prepared statement */ final StringBuilder statement = new StringBuilder("INSERT INTO `") @@ -37,6 +94,579 @@ public interface DataMapper { return statement.toString(); } + /** + * Map the inspected schema to either an existing view/table and append e.g. column or (if not existing) create a new view/table. + * @param database The database. + * @param resultSet The inspected schema. + * @return The database containing the updated view/table. + * @throws SQLException + */ + default ViewDto schemaResultSetToView(DatabaseDto database, ResultSet resultSet) throws SQLException { + return ViewDto.builder() + .name(resultSet.getString(1)) + .internalName(resultSet.getString(1)) + .vdbid(database.getId()) + .database(database) + .isInitialView(false) + .isPublic(database.getIsPublic()) + .query(resultSet.getString(9)) + .queryHash(Hashing.sha256() + .hashString(resultSet.getString(9), StandardCharsets.UTF_8) + .toString()) + .columns(new LinkedList<>()) + .identifiers(new LinkedList<>()) + .creator(database.getOwner()) + .createdBy(database.getOwner().getId()) + .build(); + } + + default TableStatisticDto resultSetToTableStatistic(ResultSet data) throws SQLException { + final TableStatisticDto statistic = TableStatisticDto.builder() + .columns(new LinkedHashMap<>()) + .build(); + while (data.next()) { + final ColumnStatisticDto columnStatistic = ColumnStatisticDto.builder() + .min(data.getBigDecimal(2)) + .max(data.getBigDecimal(3)) + .median(data.getBigDecimal(4)) + .mean(data.getBigDecimal(5)) + .stdDev(data.getBigDecimal(6)) + .build(); + statistic.getColumns().put(data.getString(1), columnStatistic); + } + return statistic; + } + + default TableDto resultSetToTable(ResultSet resultSet, TableDto table, QueryConfig queryConfig) throws SQLException { + final ColumnDto column = ColumnDto.builder() + .ordinalPosition(resultSet.getInt(1) - 1) /* start at zero */ + .autoGenerated(resultSet.getString(2) != null && resultSet.getString(2).startsWith("nextval")) + .isNullAllowed(resultSet.getString(3).equals("YES")) + .columnType(ColumnTypeDto.valueOf(resultSet.getString(4).toUpperCase())) + .d(resultSet.getString(7) != null ? resultSet.getLong(7) : null) + .name(resultSet.getString(10)) + .internalName(resultSet.getString(10)) + .table(table) + .tableId(table.getId()) + .databaseId(table.getTdbid()) + .description(resultSet.getString(11)) + .build(); + if (column.getColumnType().equals(ColumnTypeDto.ENUM)) { + column.setEnums(Arrays.stream(resultSet.getString(8) + .substring(0, resultSet.getString(8).length() - 1) + .replace("enum(", "") + .split(",")) + .map(value -> value.replace("'", "")) + .toList()); + } + if (column.getColumnType().equals(ColumnTypeDto.SET)) { + column.setSets(Arrays.stream(resultSet.getString(8) + .substring(0, resultSet.getString(8).length() - 1) + .replace("set(", "") + .split(",")) + .map(value -> value.replace("'", "")) + .toList()); + } + /* constraints */ + if (resultSet.getString(9) != null && resultSet.getString(9).equals("PRI")) { + table.getConstraints().getPrimaryKey().add(PrimaryKeyDto.builder() + .table(tableDtoToTableBriefDto(table)) + .column(columnDtoToColumnBriefDto(column)) + .build()); + } + /* fix boolean and set size for others */ + if (resultSet.getString(8).equalsIgnoreCase("tinyint(1)")) { + column.setColumnType(ColumnTypeDto.BOOL); + } else if (resultSet.getString(5) != null) { + column.setSize(resultSet.getLong(5)); + } else if (resultSet.getString(6) != null) { + column.setSize(resultSet.getLong(6)); + } + if (column.getColumnType().equals(ColumnTypeDto.TIMESTAMP) || column.getColumnType().equals(ColumnTypeDto.DATETIME)) { + column.setDateFormat(ImageDateDto.builder() + .id(queryConfig.getDefaultTimestampFormatId()) + .build()); + } else if (column.getColumnType().equals(ColumnTypeDto.DATE)) { + column.setDateFormat(ImageDateDto.builder() + .id(queryConfig.getDefaultDateFormatId()) + .build()); + } else if (column.getColumnType().equals(ColumnTypeDto.TIME)) { + column.setDateFormat(ImageDateDto.builder() + .id(queryConfig.getDefaultTimeFormatId()) + .build()); + } + table.getColumns() + .add(column); + return table; + } + + default ViewDto resultSetToTable(ResultSet resultSet, ViewDto view, QueryConfig queryConfig) throws SQLException { + final ViewColumnDto column = ViewColumnDto.builder() + .ordinalPosition(resultSet.getInt(1) - 1) /* start at zero */ + .autoGenerated(resultSet.getString(2) != null && resultSet.getString(2).startsWith("nextval")) + .isNullAllowed(resultSet.getString(3).equals("YES")) + .columnType(ColumnTypeDto.valueOf(resultSet.getString(4).toUpperCase())) + .d(resultSet.getString(7) != null ? resultSet.getLong(7) : null) + .name(resultSet.getString(10)) + .internalName(resultSet.getString(10)) + .databaseId(view.getDatabase().getId()) + .build(); + /* fix boolean and set size for others */ + if (resultSet.getString(8).equalsIgnoreCase("tinyint(1)")) { + column.setColumnType(ColumnTypeDto.BOOL); + } else if (resultSet.getString(5) != null) { + column.setSize(resultSet.getLong(5)); + } else if (resultSet.getString(6) != null) { + column.setSize(resultSet.getLong(6)); + } + if (column.getColumnType().equals(ColumnTypeDto.TIMESTAMP) || column.getColumnType().equals(ColumnTypeDto.DATETIME)) { + column.setDateFormat(ImageDateDto.builder() + .id(queryConfig.getDefaultTimestampFormatId()) + .build()); + } else if (column.getColumnType().equals(ColumnTypeDto.DATE)) { + column.setDateFormat(ImageDateDto.builder() + .id(queryConfig.getDefaultDateFormatId()) + .build()); + } else if (column.getColumnType().equals(ColumnTypeDto.TIME)) { + column.setDateFormat(ImageDateDto.builder() + .id(queryConfig.getDefaultTimeFormatId()) + .build()); + } + view.getColumns() + .add(column); + log.trace("parsed view {}.{} column: {}", view.getDatabase().getInternalName(), view.getInternalName(), column.getInternalName()); + return view; + } + + /** + * Parse columns from a SQL statement of a known database. + * @param database The database. + * @param query The SQL statement. + * @return The list of columns. + * @throws JSQLParserException The table/view or column was not found in the database. + */ + default List<ColumnDto> parseColumns(DatabaseDto database, String query) throws JSQLParserException { + final List<ColumnDto> columns = new ArrayList<>(); + final CCJSqlParserManager parserRealSql = new CCJSqlParserManager(); + final net.sf.jsqlparser.statement.Statement statement = parserRealSql.parse(new StringReader(query)); + log.trace("parse columns from query: {}", query); + /* bi-directional mapping */ + database.getTables() + .forEach(table -> table.getColumns() + .forEach(column -> column.setTable(table))); + /* check */ + if (!(statement instanceof Select selectStatement)) { + log.error("Query attempts to update the dataset, not a SELECT statement"); + throw new JSQLParserException("Query attempts to update the dataset"); + } + /* start parsing */ + final PlainSelect ps = (PlainSelect) selectStatement.getSelectBody(); + final List<SelectItem> clauses = ps.getSelectItems(); + log.trace("columns referenced in the from-clause: {}", clauses); + /* Parse all tables */ + final List<FromItem> fromItems = new ArrayList<>(fromItemToFromItems(ps.getFromItem())); + if (ps.getJoins() != null && !ps.getJoins().isEmpty()) { + log.trace("query contains join items: {}", ps.getJoins()); + for (net.sf.jsqlparser.statement.select.Join j : ps.getJoins()) { + if (j.getRightItem() != null) { + fromItems.add(j.getRightItem()); + } + } + } + final List<ColumnDto> allColumns = Stream.of(database.getViews() + .stream() + .map(ViewDto::getColumns) + .flatMap(List::stream) + .map(this::viewColumnDtoToColumnDto), + database.getTables() + .stream() + .map(TableDto::getColumns) + .flatMap(List::stream)) + .flatMap(i -> i) + .toList(); + log.trace("columns referenced in the from-clause and join-clause(s): {}", clauses); + /* Checking if all columns exist */ + for (SelectItem clause : clauses) { + final SelectExpressionItem item = (SelectExpressionItem) clause; + final Column column = (Column) item.getExpression(); + final Optional<net.sf.jsqlparser.schema.Table> optional = fromItems.stream() + .map(t -> (net.sf.jsqlparser.schema.Table) t) + .filter(t -> { + if (column.getTable() == null) { + /* column does not reference a specific table, so there is only one table */ + final String tableName = ((net.sf.jsqlparser.schema.Table) fromItems.get(0)).getName().replace("`", ""); + return tableMatches(t, tableName); + } + final String tableName = column.getTable().getName().replace("`", ""); + return tableMatches(t, tableName); + }) + .findFirst(); + if (optional.isEmpty()) { + log.error("Failed to find table/view {} (with designator {})", column.getTable().getName(), column.getTable().getAlias()); + throw new JSQLParserException("Failed to find table/view " + column.getTable().getName() + " (with alias " + column.getTable().getAlias() + ")"); + } + final String columnName = column.getColumnName().replace("`", ""); + final String tableOrView = optional.get().getName().replace("`", ""); + final List<ColumnDto> filteredColumns = allColumns.stream() + .filter(c -> (c.getAlias() != null && c.getAlias().equals(columnName)) || c.getInternalName().equals(columnName)) + .toList(); + final Optional<ColumnDto> optionalColumn = filteredColumns.stream() + .filter(c -> columnMatches(c, tableOrView)) + .findFirst(); + if (optionalColumn.isEmpty()) { + log.error("Failed to find column with name {} of table/view {} in {}", columnName, tableOrView, filteredColumns.stream().map(c -> c.getTable().getInternalName() + "." + c.getInternalName()).toList()); + throw new JSQLParserException("Failed to find column with name " + columnName + " of table/view " + tableOrView); + } + final ColumnDto resultColumn = optionalColumn.get(); + if (item.getAlias() != null) { + resultColumn.setAlias(item.getAlias().getName().replace("`", "")); + } + resultColumn.setDatabaseId(database.getId()); + resultColumn.setTable(resultColumn.getTable()); + resultColumn.setTableId(resultColumn.getTable().getId()); + log.trace("found column with internal name {} and alias {}", resultColumn.getInternalName(), resultColumn.getAlias()); + columns.add(resultColumn); + } + return columns; + } + + default boolean tableMatches(net.sf.jsqlparser.schema.Table table, String otherTableName) { + final String tableName = table.getName() + .trim() + .replace("`", ""); + if (table.getAlias() == null) { + /* table does not have designator */ + log.trace("table '{}' has no designator", tableName); + return tableName.equals(otherTableName); + } + /* has designator */ + final String designator = table.getAlias() + .getName() + .trim() + .replace("`", ""); + log.trace("table '{}' has designator {}", tableName, designator); + return designator.equals(otherTableName); + } + + default boolean columnMatches(ColumnDto column, String tableOrView) { + if (column.getTable() != null && column.getTable().getInternalName().equals(tableOrView)) { + log.trace("table '{}' found in column table", tableOrView); + return true; + } + if (column.getViews() == null) { + log.trace("table/view '{}' not found among column views: empty list", tableOrView); + return false; + } + /* maybe matches one of the other views */ + final boolean found = column.getViews() + .stream() + .anyMatch(v -> v.getInternalName().equals(tableOrView)); + if (!found) { + log.trace("table/view '{}' not found among column views: {}", tableOrView, column.getViews().stream().map(ViewDto::getInternalName).toList()); + } + return found; + } + + default List<FromItem> fromItemToFromItems(FromItem data) throws JSQLParserException { + return fromItemToFromItems(data, 0); + } + + default List<FromItem> fromItemToFromItems(FromItem data, Integer level) throws JSQLParserException { + final List<FromItem> fromItems = new LinkedList<>(); + if (data instanceof net.sf.jsqlparser.schema.Table table) { + fromItems.add(data); + log.trace("from-item {} is of type table: level ~> {}", table.getName(), level); + return fromItems; + } + if (data instanceof SubJoin subJoin) { + log.trace("from-item is of type sub-join: level ~> {}", level); + for (Join join : subJoin.getJoinList()) { + final List<FromItem> tmp = fromItemToFromItems(join.getRightItem(), level + 1); + if (tmp == null) { + log.error("Failed to find right sub-join table: {}", join.getRightItem()); + throw new JSQLParserException("Failed to find right sub-join table"); + } + fromItems.addAll(tmp); + } + final List<FromItem> tmp = fromItemToFromItems(subJoin.getLeft(), level + 1); + if (tmp == null) { + log.error("Failed to find left sub-join table: {}", subJoin.getLeft()); + throw new JSQLParserException("Failed to find left sub-join table"); + } + fromItems.addAll(tmp); + return fromItems; + } + log.warn("unknown from-item {}", data); + return null; + } + + default QueryDto resultSetToQueryDto(@NotNull ResultSet data) throws SQLException, QueryNotFoundException { + /* note that next() is called outside this mapping function */ + return QueryDto.builder() + .id(data.getLong(1)) + .created(LocalDateTime.parse(data.getString(2), mariaDbFormatter) + .atZone(ZoneId.of("UTC")) + .toInstant()) + .createdBy(UUID.fromString(data.getString(3))) + .query(data.getString(4)) + .queryHash(data.getString(5)) + .resultHash(data.getString(6)) + .resultNumber(data.getLong(7)) + .isPersisted(data.getBoolean(8)) + .execution(LocalDateTime.parse(data.getString(9), mariaDbFormatter) + .atZone(ZoneId.of("UTC")) + .toInstant()) + .build(); + } + + default List<TableHistoryDto> resultSetToTableHistory(ResultSet resultSet) throws SQLException { + /* columns */ + final List<TableHistoryDto> history = new LinkedList<>(); + while (resultSet.next()) { + history.add(TableHistoryDto.builder() + .timestamp(LocalDateTime.parse(resultSet.getString(1), mariaDbFormatter) + .atZone(ZoneId.of("UTC")) + .toInstant()) + .event(resultSet.getString(2)) + .total(resultSet.getLong(3)) + .build()); + } + log.trace("found {} history event(s)", history.size()); + return history; + } + + default TableDto resultSetToConstraint(ResultSet resultSet, TableDto table) throws SQLException { + final String type = resultSet.getString(2); + final String name = resultSet.getString(3); + final String columnName = resultSet.getString(4); + final String referencedTable = resultSet.getString(5); + final String referencedColumnName = resultSet.getString(6); + final ReferenceTypeDto deleteRule = resultSet.getString(7) != null ? ReferenceTypeDto.fromType(resultSet.getString(7)) : null; + final ReferenceTypeDto updateRule = resultSet.getString(8) != null ? ReferenceTypeDto.fromType(resultSet.getString(8)) : null; + final Optional<ColumnDto> optional = table.getColumns().stream() + .filter(c -> c.getInternalName().equals(columnName)) + .findFirst(); + if (optional.isEmpty()) { + log.error("Failed to find table column: {}", columnName); + throw new IllegalArgumentException("Failed to find table column"); + } + final ColumnDto column = optional.get(); + if (type.equals("FOREIGN KEY") || type.equals("UNIQUE")) { + final Optional<UniqueDto> optional2 = table.getConstraints().getUniques().stream().filter(u -> u.getName().equals(name)).findFirst(); + if (optional2.isPresent()) { + optional2.get() + .getColumns() + .add(column); + return table; + } + if (type.equals("UNIQUE")) { + table.getConstraints() + .getUniques() + .add(UniqueDto.builder() + .name(name) + .columns(new LinkedList<>(List.of(column))) + .build()); + return table; + } + final Optional<ForeignKeyDto> optional1 = table.getConstraints() + .getForeignKeys() + .stream() + .filter(fk -> fk.getName().equals(name)) + .findFirst(); + final ForeignKeyReferenceDto foreignKeyReference = ForeignKeyReferenceDto.builder() + .column(ColumnBriefDto.builder() + .name(columnName) + .internalName(columnName) + .databaseId(table.getTdbid()) + .build()) + .referencedColumn(ColumnBriefDto.builder() + .name(referencedColumnName) + .internalName(referencedColumnName) + .databaseId(table.getTdbid()) + .build()) + .build(); + if (optional1.isPresent()) { + foreignKeyReference.setForeignKey(foreignKeyDtoToForeignKeyBriefDto(optional1.get())); + optional1.get() + .getReferences() + .add(foreignKeyReference); + log.debug("found foreign key: create part ({}) referencing table {} ({})", columnName, referencedTable, referencedColumnName); + return table; + } + final ForeignKeyDto foreignKey = ForeignKeyDto.builder() + .name(name) + .table(tableDtoToTableBriefDto(table)) + .referencedTable(TableBriefDto.builder() + .name(referencedTable) + .internalName(referencedTable) + .databaseId(table.getTdbid()) + .build()) + .references(new LinkedList<>(List.of(foreignKeyReference))) + .onDelete(deleteRule) + .onUpdate(updateRule) + .build(); + foreignKey.getReferences() + .forEach(ref -> ref.setForeignKey(foreignKeyDtoToForeignKeyBriefDto(foreignKey))); + table.getConstraints() + .getForeignKeys() + .add(foreignKey); + log.debug("create foreign key: add part ({}) referencing table {} ({})", columnName, referencedTable, referencedColumnName); + return table; + } + return table; + } + + default TableDto schemaResultSetToTable(DatabaseDto database, ResultSet resultSet) throws SQLException, + TableNotFoundException { + if (!resultSet.next()) { + throw new TableNotFoundException("Failed to find table in the information schema"); + } + final TableDto table = TableDto.builder() + .name(resultSet.getString(1)) + .internalName(resultSet.getString(1)) + .isVersioned(resultSet.getString(2).equals("SYSTEM VERSIONED")) + .numRows(resultSet.getLong(3)) + .avgRowLength(resultSet.getLong(4)) + .dataLength(resultSet.getLong(5)) + .maxDataLength(resultSet.getLong(6)) + .tdbid(database.getId()) + .queueName("dbrepo") + .routingKey("dbrepo") + .description(resultSet.getString(10)) + .columns(new LinkedList<>()) + .identifiers(new LinkedList<>()) + .creator(database.getOwner()) + .createdBy(database.getOwner().getId()) + .owner(database.getOwner()) + .constraints(ConstraintsDto.builder() + .foreignKeys(new LinkedList<>()) + .primaryKey(new LinkedHashSet<>()) + .uniques(new LinkedList<>()) + .checks(new LinkedHashSet<>()) + .build()) + .isPublic(database.getIsPublic()) + .build(); + if (resultSet.getString(7) != null && !resultSet.getString(7).isEmpty()) { + table.setCreated(Timestamp.valueOf(resultSet.getString(7)) + .toInstant()); + } + return table; + } + + default Object dataColumnToObject(Object data, ColumnDto column) { + if (data == null) { + return null; + } + /* boolean encoding fix */ + if (column.getColumnType().equals(ColumnTypeDto.TINYINT) && column.getSize() == 1) { + log.trace("column {} is of type tinyint with size {}: map to boolean", column.getInternalName(), column.getSize()); + column.setColumnType(ColumnTypeDto.BOOL); + } + switch (column.getColumnType()) { + case DATE -> { + if (column.getDateFormat() == null) { + log.error("Missing date format for column {}", column.getId()); + throw new IllegalArgumentException("Missing date format"); + } + log.trace("mapping {} to date with format '{}'", data, column.getDateFormat()); + final DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .parseCaseInsensitive() /* case insensitive to parse JAN and FEB */ + .appendPattern(column.getDateFormat().getUnixFormat()) + .toFormatter(Locale.ENGLISH); + final LocalDate date = LocalDate.parse(String.valueOf(data), formatter); + return date.atStartOfDay(ZoneId.of("UTC")) + .toInstant(); + } + case TIMESTAMP, DATETIME -> { + if (column.getDateFormat() == null) { + log.error("Missing date format for column {}", column.getId()); + throw new IllegalArgumentException("Missing date format"); + } + log.trace("mapping {} to timestamp with format '{}'", data, column.getDateFormat()); + return Timestamp.valueOf(data.toString()) + .toInstant(); + } + case BINARY, VARBINARY, BIT -> { + log.trace("mapping {} -> binary", data); + return Long.parseLong(String.valueOf(data), 2); + } + case TEXT, CHAR, VARCHAR, TINYTEXT, MEDIUMTEXT, LONGTEXT, ENUM, SET -> { + log.trace("mapping {} -> string", data); + return String.valueOf(data); + } + case BIGINT -> { + log.trace("mapping {} -> biginteger", data); + return new BigInteger(String.valueOf(data)); + } + case INT, SMALLINT, MEDIUMINT, TINYINT -> { + log.trace("mapping {} -> integer", data); + return Integer.parseInt(String.valueOf(data)); + } + case DECIMAL, FLOAT, DOUBLE -> { + log.trace("mapping {} -> double", data); + return Double.valueOf(String.valueOf(data)); + } + case BOOL -> { + log.trace("mapping {} -> boolean", data); + return Boolean.valueOf(String.valueOf(data)); + } + case TIME -> { + log.trace("mapping {} -> time", data); + return String.valueOf(data); + } + case YEAR -> { + final String date = String.valueOf(data); + log.trace("mapping {} -> year", date); + return Short.valueOf(date.substring(0, date.indexOf('-'))); + } + } + log.warn("column type {} is not known", column.getColumnType()); + throw new IllegalArgumentException("Column type not known"); + } + + default QueryResultDto resultListToQueryResultDto(List<ColumnDto> columns, ResultSet result) throws SQLException { + log.trace("mapping result list to query result, columns.size={}", columns.size()); + final List<Map<String, Object>> resultList = new LinkedList<>(); + while (result.next()) { + /* map the result set to the columns through the stored metadata in the metadata database */ + int[] idx = new int[]{1}; + final Map<String, Object> map = new HashMap<>(); + for (final ColumnDto column : columns) { + final String columnOrAlias; + if (column.getAlias() != null) { + log.debug("column {} has alias {}", column.getInternalName(), column.getAlias()); + columnOrAlias = column.getAlias(); + } else { + columnOrAlias = column.getInternalName(); + } + if (List.of(ColumnTypeDto.BLOB, ColumnTypeDto.TINYBLOB, ColumnTypeDto.MEDIUMBLOB, ColumnTypeDto.LONGBLOB).contains(column.getColumnType())) { + log.trace("column {} is of type {}", columnOrAlias, column.getColumnType().getType().toLowerCase()); + final Blob blob = result.getBlob(idx[0]++); + final String value = blob == null ? null : Hex.encodeHexString(blob.getBytes(1, (int) blob.length())).toUpperCase(); + map.put(columnOrAlias, value); + continue; + } + final Object object = dataColumnToObject(result.getObject(idx[0]++), column); + if (object == null) { + log.warn("result set for column {} is empty (=null)", column.getInternalName()); + } + map.put(columnOrAlias, object); + } + resultList.add(map); + } + final int[] idx = new int[]{0}; + final List<Map<String, Integer>> headers = columns.stream() + .map(c -> (Map<String, Integer>) new LinkedHashMap<String, Integer>() {{ + put(c.getAlias() != null ? c.getAlias() : c.getInternalName(), idx[0]++); + }}) + .toList(); + log.trace("created ordered header list: {}", headers); + return QueryResultDto.builder() + .result(resultList) + .headers(headers) + .build(); + } + default void prepareStatementWithColumnTypeObject(PreparedStatement ps, ColumnTypeDto columnType, int idx, Object value) throws SQLException { switch (columnType) { case BLOB, TINYBLOB, MEDIUMBLOB, LONGBLOB: diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java index afb0701455515758c8bec86eac138bdccb151a20..12bf9f177fa0e0a58f88694457eee7ecc55cc94d 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java @@ -1,41 +1,16 @@ package at.tuwien.mapper; -import at.tuwien.api.container.image.ImageDateDto; -import at.tuwien.api.database.DatabaseDto; -import at.tuwien.api.database.ViewColumnDto; -import at.tuwien.api.database.ViewDto; import at.tuwien.api.database.query.ImportCsvDto; -import at.tuwien.api.database.query.QueryDto; -import at.tuwien.api.database.query.QueryResultDto; import at.tuwien.api.database.table.*; import at.tuwien.api.database.table.columns.*; -import at.tuwien.api.database.table.constraints.ConstraintsDto; -import at.tuwien.api.database.table.constraints.foreign.ForeignKeyBriefDto; -import at.tuwien.api.database.table.constraints.foreign.ForeignKeyDto; -import at.tuwien.api.database.table.constraints.foreign.ForeignKeyReferenceDto; -import at.tuwien.api.database.table.constraints.foreign.ReferenceTypeDto; -import at.tuwien.api.database.table.constraints.primary.PrimaryKeyDto; -import at.tuwien.api.database.table.constraints.unique.UniqueDto; import at.tuwien.api.database.table.internal.PrivilegedTableDto; -import at.tuwien.config.QueryConfig; import at.tuwien.exception.*; import at.tuwien.utils.MariaDbUtil; -import com.github.dockerjava.zerodep.shaded.org.apache.commons.codec.binary.Hex; -import com.google.common.hash.Hashing; -import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.parser.CCJSqlParserManager; -import net.sf.jsqlparser.schema.Column; -import net.sf.jsqlparser.statement.select.*; -import org.jetbrains.annotations.NotNull; import org.mapstruct.Mapper; -import org.mapstruct.Mapping; -import org.mapstruct.Mappings; import org.mapstruct.Named; -import javax.swing.table.TableColumn; import java.io.*; import java.math.BigInteger; -import java.nio.charset.StandardCharsets; import java.sql.*; import java.sql.Date; import java.text.Normalizer; @@ -45,9 +20,8 @@ import java.time.format.DateTimeFormatterBuilder; import java.util.*; import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; -@Mapper(componentModel = "spring", uses = {MetadataMapper.class}) +@Mapper(componentModel = "spring", uses = {MetadataMapper.class, DataMapper.class}) public interface MariaDbMapper { org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(MariaDbMapper.class); @@ -108,49 +82,6 @@ public interface MariaDbMapper { return statement.toString(); } - default QueryResultDto resultListToQueryResultDto(List<ColumnDto> columns, ResultSet result) throws SQLException { - log.trace("mapping result list to query result, columns.size={}", columns.size()); - final List<Map<String, Object>> resultList = new LinkedList<>(); - while (result.next()) { - /* map the result set to the columns through the stored metadata in the metadata database */ - int[] idx = new int[]{1}; - final Map<String, Object> map = new HashMap<>(); - for (final ColumnDto column : columns) { - final String columnOrAlias; - if (column.getAlias() != null) { - log.debug("column {} has alias {}", column.getInternalName(), column.getAlias()); - columnOrAlias = column.getAlias(); - } else { - columnOrAlias = column.getInternalName(); - } - if (List.of(ColumnTypeDto.BLOB, ColumnTypeDto.TINYBLOB, ColumnTypeDto.MEDIUMBLOB, ColumnTypeDto.LONGBLOB).contains(column.getColumnType())) { - log.trace("column {} is of type {}", columnOrAlias, column.getColumnType().getType().toLowerCase()); - final Blob blob = result.getBlob(idx[0]++); - final String value = blob == null ? null : Hex.encodeHexString(blob.getBytes(1, (int) blob.length())).toUpperCase(); - map.put(columnOrAlias, value); - continue; - } - final Object object = dataColumnToObject(result.getObject(idx[0]++), column); - if (object == null) { - log.warn("result set for column {} is empty (=null)", column.getInternalName()); - } - map.put(columnOrAlias, object); - } - resultList.add(map); - } - final int[] idx = new int[]{0}; - final List<Map<String, Integer>> headers = columns.stream() - .map(c -> (Map<String, Integer>) new LinkedHashMap<String, Integer>() {{ - put(c.getAlias() != null ? c.getAlias() : c.getInternalName(), idx[0]++); - }}) - .toList(); - log.trace("created ordered header list: {}", headers); - return QueryResultDto.builder() - .result(resultList) - .headers(headers) - .build(); - } - default String databaseTablesSelectRawQuery() { final String statement = "SELECT DISTINCT t.`TABLE_NAME` FROM information_schema.TABLES t WHERE t.`TABLE_SCHEMA` = ? AND t.`TABLE_TYPE` = 'SYSTEM VERSIONED' AND t.`TABLE_NAME` != 'qs_queries' ORDER BY t.`TABLE_NAME` ASC"; log.trace("mapped select tables statement: {}", statement); @@ -182,7 +113,7 @@ public interface MariaDbMapper { } default String databaseTableConstraintsSelectRawQuery() { - final String statement = "SELECT k.`ORDINAL_POSITION`, c.`CONSTRAINT_TYPE`, k.`CONSTRAINT_NAME`, k.`COLUMN_NAME`, k.`REFERENCED_TABLE_NAME`, k.`REFERENCED_COLUMN_NAME`, r.`DELETE_RULE`, r.`UPDATE_RULE` FROM information_schema.TABLE_CONSTRAINTS c JOIN information_schema.KEY_COLUMN_USAGE k ON c.`TABLE_NAME` = k.`TABLE_NAME` AND c.`CONSTRAINT_NAME` = k.`CONSTRAINT_NAME` LEFT JOIN information_schema.REFERENTIAL_CONSTRAINTS r ON r.`CONSTRAINT_NAME` = k.`CONSTRAINT_NAME` WHERE LOWER(k.`COLUMN_NAME`) != 'row_end' AND c.`TABLE_SCHEMA` = ? AND c.`TABLE_NAME` = ? GROUP BY k.`ORDINAL_POSITION`, k.`CONSTRAINT_NAME` ORDER BY k.`ORDINAL_POSITION` ASC;"; + final String statement = "SELECT k.`ORDINAL_POSITION`, c.`CONSTRAINT_TYPE`, k.`CONSTRAINT_NAME`, k.`COLUMN_NAME`, k.`REFERENCED_TABLE_NAME`, k.`REFERENCED_COLUMN_NAME`, r.`DELETE_RULE`, r.`UPDATE_RULE`FROM information_schema.TABLE_CONSTRAINTS c JOIN information_schema.KEY_COLUMN_USAGE k ON c.`TABLE_NAME` = k.`TABLE_NAME` AND c.`CONSTRAINT_NAME` = k.`CONSTRAINT_NAME` LEFT JOIN information_schema.REFERENTIAL_CONSTRAINTS r ON r.`CONSTRAINT_NAME` = k.`CONSTRAINT_NAME` AND r.`CONSTRAINT_SCHEMA` = c.`TABLE_NAME`WHERE LOWER(k.`COLUMN_NAME`) != 'row_end' AND c.`TABLE_SCHEMA` = ? AND c.`TABLE_NAME` = ? ORDER BY k.`ORDINAL_POSITION` ASC;"; log.trace("mapped select table constraints statement: {}", statement); return statement; } @@ -411,23 +342,6 @@ public interface MariaDbMapper { return data.getLong(1); } - default TableStatisticDto resultSetToTableStatistic(ResultSet data) throws SQLException { - final TableStatisticDto statistic = TableStatisticDto.builder() - .columns(new LinkedHashMap<>()) - .build(); - while (data.next()) { - final ColumnStatisticDto columnStatistic = ColumnStatisticDto.builder() - .min(data.getBigDecimal(2)) - .max(data.getBigDecimal(3)) - .median(data.getBigDecimal(4)) - .mean(data.getBigDecimal(5)) - .stdDev(data.getBigDecimal(6)) - .build(); - statistic.getColumns().put(data.getString(1), columnStatistic); - } - return statistic; - } - /** * Selects the dataset page from a table/view. * @@ -490,48 +404,6 @@ public interface MariaDbMapper { return "DROP TABLE `" + tableName + "`;"; } - default String tupleToRawInsertQuery(PrivilegedTableDto table, TupleDto data) throws TableMalformedException { - log.trace("mapping table data to insert query, table={}, data={}", table, data); - if (table.getColumns().isEmpty()) { - throw new TableMalformedException("Columns are not known: empty"); - } - /* parameterized query for prepared statement */ - final StringBuilder statement = new StringBuilder("INSERT INTO `") - .append(table.getInternalName()) - .append("` (") - .append(data.getData() - .keySet() - .stream() - .map(o -> "`" + o + "`") - .collect(Collectors.joining(","))) - .append(") VALUES (") - .append(data.getData() - .keySet() - .stream() - .map(o -> "?") - .collect(Collectors.joining(","))); - statement.append(");"); - for (int i = 0; i < table.getColumns().size(); i++) { - final ColumnDto column = table.getColumns() - .get(i); - if (column.getAutoGenerated()) { - log.trace("column is auto-generated, skip."); - continue; - } - final Optional<Map.Entry<String, Object>> tuple = data.getData() - .entrySet() - .stream() - .filter(d -> d.getKey().equals(column.getInternalName())) - .findFirst(); - if (tuple.isEmpty()) { - log.error("Failed to map column name {}, known names: {}", column.getInternalName(), data.getData().keySet()); - throw new TableMalformedException("Failed to map column names: not all columns are present in the tuple!"); - } - } - log.trace("mapped tuple insert query: {}", statement); - return statement.toString(); - } - default String tableOrViewToRawExportQuery(String databaseName, String tableOrView, List<ColumnDto> columns, Instant timestamp, String filePath) { final StringBuilder statement = new StringBuilder("SELECT "); @@ -583,280 +455,6 @@ public interface MariaDbMapper { return statement.toString(); } - /** - * Map the inspected schema to either an existing view/table and append e.g. column or (if not existing) create a new view/table. - * @param database The database. - * @param resultSet The inspected schema. - * @return The database containing the updated view/table. - * @throws SQLException - */ - default ViewDto schemaResultSetToView(DatabaseDto database, ResultSet resultSet) throws SQLException { - return ViewDto.builder() - .name(resultSet.getString(1)) - .internalName(resultSet.getString(1)) - .vdbid(database.getId()) - .database(database) - .isInitialView(false) - .isPublic(database.getIsPublic()) - .query(resultSet.getString(9)) - .queryHash(Hashing.sha256() - .hashString(resultSet.getString(9), StandardCharsets.UTF_8) - .toString()) - .columns(new LinkedList<>()) - .identifiers(new LinkedList<>()) - .creator(database.getOwner()) - .createdBy(database.getOwner().getId()) - .build(); - } - - ViewColumnDto columnDtoToViewColumnDto(ColumnDto data); - - ColumnDto viewColumnDtoToColumnDto(ViewColumnDto data); - - default TableDto schemaResultSetToTable(DatabaseDto database, ResultSet resultSet) throws SQLException, - TableNotFoundException { - if (!resultSet.next()) { - throw new TableNotFoundException("Failed to find table in the information schema"); - } - final TableDto table = TableDto.builder() - .name(resultSet.getString(1)) - .internalName(resultSet.getString(1)) - .isVersioned(resultSet.getString(2).equals("SYSTEM VERSIONED")) - .numRows(resultSet.getLong(3)) - .avgRowLength(resultSet.getLong(4)) - .dataLength(resultSet.getLong(5)) - .maxDataLength(resultSet.getLong(6)) - .tdbid(database.getId()) - .queueName("dbrepo") - .routingKey("dbrepo") - .description(resultSet.getString(10)) - .columns(new LinkedList<>()) - .identifiers(new LinkedList<>()) - .creator(database.getOwner()) - .createdBy(database.getOwner().getId()) - .owner(database.getOwner()) - .constraints(ConstraintsDto.builder() - .foreignKeys(new LinkedList<>()) - .primaryKey(new LinkedHashSet<>()) - .uniques(new LinkedList<>()) - .checks(new LinkedHashSet<>()) - .build()) - .isPublic(database.getIsPublic()) - .build(); - if (resultSet.getString(7) != null && !resultSet.getString(7).isEmpty()) { - table.setCreated(Timestamp.valueOf(resultSet.getString(7)) - .toInstant()); - } - return table; - } - - ForeignKeyBriefDto foreignKeyDtoToForeignKeyBriefDto(ForeignKeyDto data); - - default TableDto resultSetToConstraint(ResultSet resultSet, TableDto table) throws SQLException { - final String type = resultSet.getString(2); - final String name = resultSet.getString(3); - final String columnName = resultSet.getString(4); - final String referencedTable = resultSet.getString(5); - final String referencedColumnName = resultSet.getString(6); - final ReferenceTypeDto deleteRule = resultSet.getString(7) != null ? ReferenceTypeDto.fromType(resultSet.getString(7)) : null; - final ReferenceTypeDto updateRule = resultSet.getString(8) != null ? ReferenceTypeDto.fromType(resultSet.getString(8)) : null; - final Optional<ColumnDto> optional = table.getColumns().stream() - .filter(c -> c.getInternalName().equals(columnName)) - .findFirst(); - if (optional.isEmpty()) { - log.error("Failed to find table column: {}", columnName); - throw new IllegalArgumentException("Failed to find table column"); - } - final ColumnDto column = optional.get(); - if (type.equals("FOREIGN KEY") || type.equals("UNIQUE")) { - final Optional<UniqueDto> optional2 = table.getConstraints().getUniques().stream().filter(u -> u.getName().equals(name)).findFirst(); - if (optional2.isPresent()) { - optional2.get() - .getColumns() - .add(column); - return table; - } - if (type.equals("UNIQUE")) { - table.getConstraints() - .getUniques() - .add(UniqueDto.builder() - .name(name) - .columns(new LinkedList<>(List.of(column))) - .build()); - return table; - } - final Optional<ForeignKeyDto> optional1 = table.getConstraints() - .getForeignKeys() - .stream() - .filter(fk -> fk.getName().equals(name)) - .findFirst(); - final ForeignKeyReferenceDto foreignKeyReference = ForeignKeyReferenceDto.builder() - .column(ColumnBriefDto.builder() - .name(columnName) - .internalName(columnName) - .databaseId(table.getTdbid()) - .build()) - .referencedColumn(ColumnBriefDto.builder() - .name(referencedColumnName) - .internalName(referencedColumnName) - .databaseId(table.getTdbid()) - .build()) - .build(); - if (optional1.isPresent()) { - foreignKeyReference.setForeignKey(foreignKeyDtoToForeignKeyBriefDto(optional1.get())); - optional1.get() - .getReferences() - .add(foreignKeyReference); - log.debug("found foreign key: create part ({}) referencing table {} ({})", columnName, referencedTable, referencedColumnName); - return table; - } - final ForeignKeyDto foreignKey = ForeignKeyDto.builder() - .name(name) - .table(tableDtoToTableBriefDto(table)) - .referencedTable(TableBriefDto.builder() - .name(referencedTable) - .internalName(referencedTable) - .databaseId(table.getTdbid()) - .build()) - .references(new LinkedList<>(List.of(foreignKeyReference))) - .onDelete(deleteRule) - .onUpdate(updateRule) - .build(); - foreignKey.getReferences() - .forEach(ref -> ref.setForeignKey(foreignKeyDtoToForeignKeyBriefDto(foreignKey))); - table.getConstraints() - .getForeignKeys() - .add(foreignKey); - log.debug("create foreign key: add part ({}) referencing table {} ({})", columnName, referencedTable, referencedColumnName); - return table; - } - return table; - } - - @Mappings({ - @Mapping(target = "databaseId", source = "tdbid") - }) - TableBriefDto tableDtoToTableBriefDto(TableDto data); - - ColumnBriefDto columnDtoToColumnBriefDto(ColumnDto data); - - default TableDto resultSetToTable(ResultSet resultSet, TableDto table, QueryConfig queryConfig) throws SQLException { - final ColumnDto column = ColumnDto.builder() - .ordinalPosition(resultSet.getInt(1) - 1) /* start at zero */ - .autoGenerated(resultSet.getString(2) != null && resultSet.getString(2).startsWith("nextval")) - .isNullAllowed(resultSet.getString(3).equals("YES")) - .columnType(ColumnTypeDto.valueOf(resultSet.getString(4).toUpperCase())) - .d(resultSet.getString(7) != null ? resultSet.getLong(7) : null) - .name(resultSet.getString(10)) - .internalName(resultSet.getString(10)) - .table(table) - .tableId(table.getId()) - .databaseId(table.getTdbid()) - .description(resultSet.getString(11)) - .build(); - if (column.getColumnType().equals(ColumnTypeDto.ENUM)) { - column.setEnums(Arrays.stream(resultSet.getString(8) - .substring(0, resultSet.getString(8).length() - 1) - .replace("enum(", "") - .split(",")) - .map(value -> value.replace("'", "")) - .toList()); - } - if (column.getColumnType().equals(ColumnTypeDto.SET)) { - column.setSets(Arrays.stream(resultSet.getString(8) - .substring(0, resultSet.getString(8).length() - 1) - .replace("set(", "") - .split(",")) - .map(value -> value.replace("'", "")) - .toList()); - } - /* constraints */ - if (resultSet.getString(9) != null && resultSet.getString(9).equals("PRI")) { - table.getConstraints().getPrimaryKey().add(PrimaryKeyDto.builder() - .table(tableDtoToTableBriefDto(table)) - .column(columnDtoToColumnBriefDto(column)) - .build()); - } - /* fix boolean and set size for others */ - if (resultSet.getString(8).equalsIgnoreCase("tinyint(1)")) { - column.setColumnType(ColumnTypeDto.BOOL); - } else if (resultSet.getString(5) != null) { - column.setSize(resultSet.getLong(5)); - } else if (resultSet.getString(6) != null) { - column.setSize(resultSet.getLong(6)); - } - if (column.getColumnType().equals(ColumnTypeDto.TIMESTAMP) || column.getColumnType().equals(ColumnTypeDto.DATETIME)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultTimestampFormatId()) - .build()); - } else if (column.getColumnType().equals(ColumnTypeDto.DATE)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultDateFormatId()) - .build()); - } else if (column.getColumnType().equals(ColumnTypeDto.TIME)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultTimeFormatId()) - .build()); - } - table.getColumns() - .add(column); - return table; - } - - default ViewDto resultSetToTable(ResultSet resultSet, ViewDto view, QueryConfig queryConfig) throws SQLException { - final ViewColumnDto column = ViewColumnDto.builder() - .ordinalPosition(resultSet.getInt(1) - 1) /* start at zero */ - .autoGenerated(resultSet.getString(2) != null && resultSet.getString(2).startsWith("nextval")) - .isNullAllowed(resultSet.getString(3).equals("YES")) - .columnType(ColumnTypeDto.valueOf(resultSet.getString(4).toUpperCase())) - .d(resultSet.getString(7) != null ? resultSet.getLong(7) : null) - .name(resultSet.getString(10)) - .internalName(resultSet.getString(10)) - .databaseId(view.getDatabase().getId()) - .build(); - /* fix boolean and set size for others */ - if (resultSet.getString(8).equalsIgnoreCase("tinyint(1)")) { - column.setColumnType(ColumnTypeDto.BOOL); - } else if (resultSet.getString(5) != null) { - column.setSize(resultSet.getLong(5)); - } else if (resultSet.getString(6) != null) { - column.setSize(resultSet.getLong(6)); - } - if (column.getColumnType().equals(ColumnTypeDto.TIMESTAMP) || column.getColumnType().equals(ColumnTypeDto.DATETIME)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultTimestampFormatId()) - .build()); - } else if (column.getColumnType().equals(ColumnTypeDto.DATE)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultDateFormatId()) - .build()); - } else if (column.getColumnType().equals(ColumnTypeDto.TIME)) { - column.setDateFormat(ImageDateDto.builder() - .id(queryConfig.getDefaultTimeFormatId()) - .build()); - } - view.getColumns() - .add(column); - log.trace("parsed view {}.{} column: {}", view.getDatabase().getInternalName(), view.getInternalName(), column.getInternalName()); - return view; - } - - default List<TableHistoryDto> resultSetToTableHistory(ResultSet resultSet) throws SQLException { - /* columns */ - final List<TableHistoryDto> history = new LinkedList<>(); - while (resultSet.next()) { - history.add(TableHistoryDto.builder() - .timestamp(LocalDateTime.parse(resultSet.getString(1), mariaDbFormatter) - .atZone(ZoneId.of("UTC")) - .toInstant()) - .event(resultSet.getString(2)) - .total(resultSet.getLong(3)) - .build()); - } - log.trace("found {} history event(s)", history.size()); - return history; - } - default String datasetToRawInsertQuery(String databaseName, PrivilegedTableDto table, ImportCsvDto data) { final StringBuilder statement = new StringBuilder("LOAD DATA INFILE '") .append(data.getLocation()) @@ -1292,258 +890,6 @@ public interface MariaDbMapper { } } - default Object dataColumnToObject(Object data, ColumnDto column) { - if (data == null) { - return null; - } - /* boolean encoding fix */ - if (column.getColumnType().equals(ColumnTypeDto.TINYINT) && column.getSize() == 1) { - log.trace("column {} is of type tinyint with size {}: map to boolean", column.getInternalName(), column.getSize()); - column.setColumnType(ColumnTypeDto.BOOL); - } - switch (column.getColumnType()) { - case DATE -> { - if (column.getDateFormat() == null) { - log.error("Missing date format for column {}", column.getId()); - throw new IllegalArgumentException("Missing date format"); - } - log.trace("mapping {} to date with format '{}'", data, column.getDateFormat()); - final DateTimeFormatter formatter = new DateTimeFormatterBuilder() - .parseCaseInsensitive() /* case insensitive to parse JAN and FEB */ - .appendPattern(column.getDateFormat().getUnixFormat()) - .toFormatter(Locale.ENGLISH); - final LocalDate date = LocalDate.parse(String.valueOf(data), formatter); - return date.atStartOfDay(ZoneId.of("UTC")) - .toInstant(); - } - case TIMESTAMP, DATETIME -> { - if (column.getDateFormat() == null) { - log.error("Missing date format for column {}", column.getId()); - throw new IllegalArgumentException("Missing date format"); - } - log.trace("mapping {} to timestamp with format '{}'", data, column.getDateFormat()); - return Timestamp.valueOf(data.toString()) - .toInstant(); - } - case BINARY, VARBINARY, BIT -> { - log.trace("mapping {} -> binary", data); - return Long.parseLong(String.valueOf(data), 2); - } - case TEXT, CHAR, VARCHAR, TINYTEXT, MEDIUMTEXT, LONGTEXT, ENUM, SET -> { - log.trace("mapping {} -> string", data); - return String.valueOf(data); - } - case BIGINT -> { - log.trace("mapping {} -> biginteger", data); - return new BigInteger(String.valueOf(data)); - } - case INT, SMALLINT, MEDIUMINT, TINYINT -> { - log.trace("mapping {} -> integer", data); - return Integer.parseInt(String.valueOf(data)); - } - case DECIMAL, FLOAT, DOUBLE -> { - log.trace("mapping {} -> double", data); - return Double.valueOf(String.valueOf(data)); - } - case BOOL -> { - log.trace("mapping {} -> boolean", data); - return Boolean.valueOf(String.valueOf(data)); - } - case TIME -> { - log.trace("mapping {} -> time", data); - return String.valueOf(data); - } - case YEAR -> { - final String date = String.valueOf(data); - log.trace("mapping {} -> year", date); - return Short.valueOf(date.substring(0, date.indexOf('-'))); - } - } - log.warn("column type {} is not known", column.getColumnType()); - throw new IllegalArgumentException("Column type not known"); - } - - /** - * Parse columns from a SQL statement of a known database. - * @param database The database. - * @param query The SQL statement. - * @return The list of columns. - * @throws JSQLParserException The table/view or column was not found in the database. - */ - default List<ColumnDto> parseColumns(DatabaseDto database, String query) throws JSQLParserException { - final List<ColumnDto> columns = new ArrayList<>(); - final CCJSqlParserManager parserRealSql = new CCJSqlParserManager(); - final net.sf.jsqlparser.statement.Statement statement = parserRealSql.parse(new StringReader(query)); - log.trace("parse columns from query: {}", query); - /* bi-directional mapping */ - database.getTables() - .forEach(table -> table.getColumns() - .forEach(column -> column.setTable(table))); - /* check */ - if (!(statement instanceof Select selectStatement)) { - log.error("Query attempts to update the dataset, not a SELECT statement"); - throw new JSQLParserException("Query attempts to update the dataset"); - } - /* start parsing */ - final PlainSelect ps = (PlainSelect) selectStatement.getSelectBody(); - final List<SelectItem> clauses = ps.getSelectItems(); - log.trace("columns referenced in the from-clause: {}", clauses); - /* Parse all tables */ - final List<FromItem> fromItems = new ArrayList<>(fromItemToFromItems(ps.getFromItem())); - if (ps.getJoins() != null && !ps.getJoins().isEmpty()) { - log.trace("query contains join items: {}", ps.getJoins()); - for (net.sf.jsqlparser.statement.select.Join j : ps.getJoins()) { - if (j.getRightItem() != null) { - fromItems.add(j.getRightItem()); - } - } - } - final List<ColumnDto> allColumns = Stream.of(database.getViews() - .stream() - .map(ViewDto::getColumns) - .flatMap(List::stream) - .map(this::viewColumnDtoToColumnDto), - database.getTables() - .stream() - .map(TableDto::getColumns) - .flatMap(List::stream)) - .flatMap(i -> i) - .toList(); - log.trace("columns referenced in the from-clause and join-clause(s): {}", clauses); - /* Checking if all columns exist */ - for (SelectItem clause : clauses) { - final SelectExpressionItem item = (SelectExpressionItem) clause; - final Column column = (Column) item.getExpression(); - final Optional<net.sf.jsqlparser.schema.Table> optional = fromItems.stream() - .map(t -> (net.sf.jsqlparser.schema.Table) t) - .filter(t -> { - if (column.getTable() == null) { - /* column does not reference a specific table, so there is only one table */ - final String tableName = ((net.sf.jsqlparser.schema.Table) fromItems.get(0)).getName().replace("`", ""); - return tableMatches(t, tableName); - } - final String tableName = column.getTable().getName().replace("`", ""); - return tableMatches(t, tableName); - }) - .findFirst(); - if (optional.isEmpty()) { - log.error("Failed to find table/view {} (with designator {})", column.getTable().getName(), column.getTable().getAlias()); - throw new JSQLParserException("Failed to find table/view " + column.getTable().getName() + " (with alias " + column.getTable().getAlias() + ")"); - } - final String columnName = column.getColumnName().replace("`", ""); - final String tableOrView = optional.get().getName().replace("`", ""); - final List<ColumnDto> filteredColumns = allColumns.stream() - .filter(c -> (c.getAlias() != null && c.getAlias().equals(columnName)) || c.getInternalName().equals(columnName)) - .toList(); - final Optional<ColumnDto> optionalColumn = filteredColumns.stream() - .filter(c -> columnMatches(c, tableOrView)) - .findFirst(); - if (optionalColumn.isEmpty()) { - log.error("Failed to find column with name {} of table/view {} in {}", columnName, tableOrView, filteredColumns.stream().map(c -> c.getTable().getInternalName() + "." + c.getInternalName()).toList()); - throw new JSQLParserException("Failed to find column with name " + columnName + " of table/view " + tableOrView); - } - final ColumnDto resultColumn = optionalColumn.get(); - if (item.getAlias() != null) { - resultColumn.setAlias(item.getAlias().getName().replace("`", "")); - } - resultColumn.setDatabaseId(database.getId()); - resultColumn.setTable(resultColumn.getTable()); - resultColumn.setTableId(resultColumn.getTable().getId()); - log.trace("found column with internal name {} and alias {}", resultColumn.getInternalName(), resultColumn.getAlias()); - columns.add(resultColumn); - } - return columns; - } - - default boolean tableMatches(net.sf.jsqlparser.schema.Table table, String otherTableName) { - final String tableName = table.getName() - .trim() - .replace("`", ""); - if (table.getAlias() == null) { - /* table does not have designator */ - log.trace("table '{}' has no designator", tableName); - return tableName.equals(otherTableName); - } - /* has designator */ - final String designator = table.getAlias() - .getName() - .trim() - .replace("`", ""); - log.trace("table '{}' has designator {}", tableName, designator); - return designator.equals(otherTableName); - } - - default boolean columnMatches(ColumnDto column, String tableOrView) { - if (column.getTable() != null && column.getTable().getInternalName().equals(tableOrView)) { - log.trace("table '{}' found in column table", tableOrView); - return true; - } - if (column.getViews() == null) { - log.trace("table/view '{}' not found among column views: empty list", tableOrView); - return false; - } - /* maybe matches one of the other views */ - final boolean found = column.getViews() - .stream() - .anyMatch(v -> v.getInternalName().equals(tableOrView)); - if (!found) { - log.trace("table/view '{}' not found among column views: {}", tableOrView, column.getViews().stream().map(ViewDto::getInternalName).toList()); - } - return found; - } - - default List<FromItem> fromItemToFromItems(FromItem data) throws JSQLParserException { - return fromItemToFromItems(data, 0); - } - - default List<FromItem> fromItemToFromItems(FromItem data, Integer level) throws JSQLParserException { - final List<FromItem> fromItems = new LinkedList<>(); - if (data instanceof net.sf.jsqlparser.schema.Table table) { - fromItems.add(data); - log.trace("from-item {} is of type table: level ~> {}", table.getName(), level); - return fromItems; - } - if (data instanceof SubJoin subJoin) { - log.trace("from-item is of type sub-join: level ~> {}", level); - for (Join join : subJoin.getJoinList()) { - final List<FromItem> tmp = fromItemToFromItems(join.getRightItem(), level + 1); - if (tmp == null) { - log.error("Failed to find right sub-join table: {}", join.getRightItem()); - throw new JSQLParserException("Failed to find right sub-join table"); - } - fromItems.addAll(tmp); - } - final List<FromItem> tmp = fromItemToFromItems(subJoin.getLeft(), level + 1); - if (tmp == null) { - log.error("Failed to find left sub-join table: {}", subJoin.getLeft()); - throw new JSQLParserException("Failed to find left sub-join table"); - } - fromItems.addAll(tmp); - return fromItems; - } - log.warn("unknown from-item {}", data); - return null; - } - - default QueryDto resultSetToQueryDto(@NotNull ResultSet data) throws SQLException, QueryNotFoundException { - /* note that next() is called outside this mapping function */ - return QueryDto.builder() - .id(data.getLong(1)) - .created(LocalDateTime.parse(data.getString(2), mariaDbFormatter) - .atZone(ZoneId.of("UTC")) - .toInstant()) - .createdBy(UUID.fromString(data.getString(3))) - .query(data.getString(4)) - .queryHash(data.getString(5)) - .resultHash(data.getString(6)) - .resultNumber(data.getLong(7)) - .isPersisted(data.getBoolean(8)) - .execution(LocalDateTime.parse(data.getString(9), mariaDbFormatter) - .atZone(ZoneId.of("UTC")) - .toInstant()) - .build(); - } - default String selectRawSelectQuery(String query, Instant timestamp, Long page, Long size) { query = query.toLowerCase(Locale.ROOT) .trim(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java index 4cde78c7d913108bdf9d3d0d1b13c541ca44724d..fca56314af224c36719d7c5b424b47df50893033 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MetadataMapper.java @@ -10,6 +10,7 @@ import at.tuwien.api.database.internal.PrivilegedDatabaseDto; import at.tuwien.api.database.internal.PrivilegedViewDto; import at.tuwien.api.database.table.TableBriefDto; import at.tuwien.api.database.table.TableDto; +import at.tuwien.api.database.table.columns.ColumnBriefDto; import at.tuwien.api.database.table.columns.ColumnDto; import at.tuwien.api.database.table.internal.PrivilegedTableDto; import at.tuwien.api.user.PrivilegedUserDto; @@ -33,9 +34,6 @@ public interface MetadataMapper { ViewColumnDto columnDtoToViewColumnDto(ColumnDto data); - /* keep */ - TableBriefDto tableDtoToTableBriefDto(TableDto data); - @Mappings({ @Mapping(target = "database", expression = "java(PrivilegedDatabaseDto.builder().container(PrivilegedContainerDto.builder().image(new ImageDto()).build()).build())") }) @@ -47,4 +45,9 @@ public interface MetadataMapper { PrivilegedUserDto userDtoToPrivilegedUserDto(UserDto data); + @Mappings({ + @Mapping(target = "databaseId", source = "tdbid") + }) + TableBriefDto tableDtoToTableBriefDto(TableDto data); + } diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java index 537c4878a4f44a1b47e39278b3233f03448cf439..cc5840080b21ae8549632db7123e12ec72b1ee30 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java @@ -1,17 +1,14 @@ package at.tuwien.service.impl; import at.tuwien.api.database.DatabaseDto; -import at.tuwien.api.database.ViewColumnDto; import at.tuwien.api.database.ViewDto; import at.tuwien.api.database.internal.PrivilegedDatabaseDto; import at.tuwien.api.database.table.TableDto; -import at.tuwien.api.database.table.columns.ColumnDto; import at.tuwien.api.database.table.constraints.unique.UniqueDto; import at.tuwien.config.QueryConfig; import at.tuwien.exception.TableNotFoundException; -import at.tuwien.exception.ViewMalformedException; import at.tuwien.exception.ViewNotFoundException; -import at.tuwien.exception.ViewSchemaException; +import at.tuwien.mapper.DataMapper; import at.tuwien.mapper.MariaDbMapper; import at.tuwien.mapper.MetadataMapper; import at.tuwien.service.SchemaService; @@ -25,20 +22,20 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.LinkedList; -import java.util.List; -import java.util.Optional; @Log4j2 @Service public class SchemaServiceMariaDbImpl extends HibernateConnector implements SchemaService { + private final DataMapper dataMapper; private final QueryConfig queryConfig; private final MariaDbMapper mariaDbMapper; private final MetadataMapper metadataMapper; @Autowired - public SchemaServiceMariaDbImpl(QueryConfig queryConfig, MariaDbMapper mariaDbMapper, + public SchemaServiceMariaDbImpl(DataMapper dataMapper, QueryConfig queryConfig, MariaDbMapper mariaDbMapper, MetadataMapper metadataMapper) { + this.dataMapper = dataMapper; this.queryConfig = queryConfig; this.mariaDbMapper = mariaDbMapper; this.metadataMapper = metadataMapper; @@ -56,7 +53,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche statement1.setString(1, database.getInternalName()); statement1.setString(2, tableName); log.trace("1={}, 2={}", database.getInternalName(), tableName); - TableDto table = mariaDbMapper.schemaResultSetToTable(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), statement1.executeQuery()); + TableDto table = dataMapper.schemaResultSetToTable(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), statement1.executeQuery()); /* obtain columns metadata */ final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery()); statement2.setString(1, database.getInternalName()); @@ -64,7 +61,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche log.trace("1={}, 2={}", database.getInternalName(), tableName); final ResultSet resultSet2 = statement2.executeQuery(); while (resultSet2.next()) { - table = mariaDbMapper.resultSetToTable(resultSet2, table, queryConfig); + table = dataMapper.resultSetToTable(resultSet2, table, queryConfig); } /* obtain check constraints metadata */ final PreparedStatement statement3 = connection.prepareStatement(mariaDbMapper.columnsCheckConstraintSelectRawQuery()); @@ -86,7 +83,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche log.trace("1={}, 2={}", database.getInternalName(), tableName); final ResultSet resultSet4 = statement4.executeQuery(); while (resultSet4.next()) { - table = mariaDbMapper.resultSetToConstraint(resultSet4, table); + table = dataMapper.resultSetToConstraint(resultSet4, table); for (UniqueDto uk : table.getConstraints().getUniques()) { uk.setTable(metadataMapper.tableDtoToTableBriefDto(table)); final TableDto tmpTable = table; @@ -133,7 +130,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche if (!resultSet1.next()) { throw new ViewNotFoundException("Failed to find view in the information schema"); } - ViewDto view = mariaDbMapper.schemaResultSetToView(metadataMapper.privilegedDatabaseDtoToDatabaseDto(privilegedDatabase), resultSet1); + ViewDto view = dataMapper.schemaResultSetToView(metadataMapper.privilegedDatabaseDtoToDatabaseDto(privilegedDatabase), resultSet1); view.setDatabase(database); view.setVdbid(database.getId()); view.setCreator(database.getCreator()); @@ -148,7 +145,7 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche .columns(new LinkedList<>()) .build(); while (resultSet2.next()) { - tmp = mariaDbMapper.resultSetToTable(resultSet2, tmp, queryConfig); + tmp = dataMapper.resultSetToTable(resultSet2, tmp, queryConfig); } view.setColumns(tmp.getColumns() .stream() diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java index d298f2fada278f1eba060b030fbcc1040324adc1..2ab2f7b349fe6cf2ba2129679a22024e9167c0fc 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java @@ -14,6 +14,7 @@ import at.tuwien.config.S3Config; import at.tuwien.exception.*; import at.tuwien.gateway.DataDatabaseSidecarGateway; import at.tuwien.gateway.MetadataServiceGateway; +import at.tuwien.mapper.DataMapper; import at.tuwien.mapper.MariaDbMapper; import at.tuwien.mapper.MetadataMapper; import at.tuwien.service.SubsetService; @@ -35,6 +36,7 @@ import java.util.UUID; public class SubsetServiceMariaDbImpl extends HibernateConnector implements SubsetService { private final S3Config s3Config; + private final DataMapper dataMapper; private final MariaDbMapper mariaDbMapper; private final MetadataMapper metadataMapper; private final StorageService storageService; @@ -42,10 +44,12 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs private final DataDatabaseSidecarGateway dataDatabaseSidecarGateway; @Autowired - public SubsetServiceMariaDbImpl(S3Config s3Config, MariaDbMapper mariaDbMapper, MetadataMapper metadataMapper, - StorageService storageService, MetadataServiceGateway metadataServiceGateway, + public SubsetServiceMariaDbImpl(S3Config s3Config, DataMapper dataMapper, MariaDbMapper mariaDbMapper, + MetadataMapper metadataMapper, StorageService storageService, + MetadataServiceGateway metadataServiceGateway, DataDatabaseSidecarGateway dataDatabaseSidecarGateway) { this.s3Config = s3Config; + this.dataMapper = dataMapper; this.mariaDbMapper = mariaDbMapper; this.metadataMapper = metadataMapper; this.storageService = storageService; @@ -97,7 +101,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs SQLException { final List<ColumnDto> columns; try { - columns = mariaDbMapper.parseColumns(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), query.getQuery()); + columns = dataMapper.parseColumns(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), query.getQuery()); } catch (JSQLParserException e) { log.error("Failed to map/parse columns: {}", e.getMessage()); throw new TableMalformedException("Failed to map/parse columns: " + e.getMessage(), e); @@ -129,7 +133,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ResultSet resultSet = statement.executeQuery(); final List<QueryDto> queries = new LinkedList<>(); while (resultSet.next()) { - final QueryDto query = mariaDbMapper.resultSetToQueryDto(resultSet); + final QueryDto query = dataMapper.resultSetToQueryDto(resultSet); query.setIdentifiers(identifiers.stream() .filter(i -> i.getType().equals(IdentifierTypeDto.SUBSET)) .filter(i -> i.getQueryId().equals(query.getId())) @@ -176,7 +180,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs try { final PreparedStatement preparedStatement = connection.prepareStatement(statement); final ResultSet resultSet = preparedStatement.executeQuery(); - return mariaDbMapper.resultListToQueryResultDto(columns, resultSet); + return dataMapper.resultListToQueryResultDto(columns, resultSet); } catch (SQLException e) { log.error("Failed to execute and map time-versioned query: {}", e.getMessage()); throw new TableMalformedException("Failed to execute and map time-versioned query: " + e.getMessage(), e); @@ -214,7 +218,7 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs if (!resultSet.next()) { throw new QueryNotFoundException("Failed to find query"); } - final QueryDto query = mariaDbMapper.resultSetToQueryDto(resultSet); + final QueryDto query = dataMapper.resultSetToQueryDto(resultSet); query.setIdentifiers(metadataServiceGateway.getIdentifiers(database.getId(), queryId)); final UserDto creator = metadataServiceGateway.getUserById(query.getCreatedBy()); log.debug("retrieved creator from metadata service: creator.id={}, creator.username={}", creator.getId(), creator.getUsername()); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java index 55e96c516185699e6c17e8e58ed43de1637da1bf..2b47e09dc424051609062927946868027a6e7392 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java @@ -13,6 +13,7 @@ import at.tuwien.api.database.table.internal.TableCreateDto; import at.tuwien.config.S3Config; import at.tuwien.exception.*; import at.tuwien.gateway.DataDatabaseSidecarGateway; +import at.tuwien.mapper.DataMapper; import at.tuwien.mapper.MariaDbMapper; import at.tuwien.service.SchemaService; import at.tuwien.service.StorageService; @@ -33,16 +34,18 @@ import java.util.*; public class TableServiceMariaDbImpl extends HibernateConnector implements TableService { private final S3Config s3Config; + private final DataMapper dataMapper; private final MariaDbMapper mariaDbMapper; private final SchemaService schemaService; private final StorageService storageService; private final DataDatabaseSidecarGateway dataDatabaseSidecarGateway; @Autowired - public TableServiceMariaDbImpl(S3Config s3Config, MariaDbMapper mariaDbMapper, SchemaService schemaService, - StorageService storageService, + public TableServiceMariaDbImpl(S3Config s3Config, DataMapper dataMapper, MariaDbMapper mariaDbMapper, + SchemaService schemaService, StorageService storageService, DataDatabaseSidecarGateway dataDatabaseSidecarGateway) { this.s3Config = s3Config; + this.dataMapper = dataMapper; this.mariaDbMapper = mariaDbMapper; this.schemaService = schemaService; this.storageService = storageService; @@ -91,7 +94,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table /* obtain statistic */ final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.tableColumnStatisticsSelectRawQuery(table.getColumns(), table.getInternalName())) .executeQuery(); - statistic = mariaDbMapper.resultSetToTableStatistic(resultSet); + statistic = dataMapper.resultSetToTableStatistic(resultSet); statistic.setRows(getCount(table, null)); } catch (SQLException e) { connection.rollback(); @@ -181,7 +184,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table timestamp, size, page)) .executeQuery(); connection.commit(); - queryResult = mariaDbMapper.resultListToQueryResultDto(table.getColumns(), resultSet); + queryResult = dataMapper.resultListToQueryResultDto(table.getColumns(), resultSet); } catch (SQLException e) { connection.rollback(); log.error("Failed to find data from table {}.{}: {}", table.getDatabase().getInternalName(), table.getInternalName(), e.getMessage()); @@ -205,7 +208,7 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectHistoryRawQuery( table.getDatabase().getInternalName(), table.getInternalName(), size)) .executeQuery(); - history = mariaDbMapper.resultSetToTableHistory(resultSet); + history = dataMapper.resultSetToTableHistory(resultSet); connection.commit(); } catch (SQLException e) { connection.rollback(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java index c85f5bfbdba9ffbe83d512a37f3f5cdebf5e4c1d..6f88c409737605671eb31e8b151e9a35ae23afa5 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java @@ -11,6 +11,7 @@ import at.tuwien.config.QueryConfig; import at.tuwien.config.S3Config; import at.tuwien.exception.*; import at.tuwien.gateway.DataDatabaseSidecarGateway; +import at.tuwien.mapper.DataMapper; import at.tuwien.mapper.MariaDbMapper; import at.tuwien.mapper.MetadataMapper; import at.tuwien.service.SchemaService; @@ -37,6 +38,7 @@ import java.util.List; public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewService { private final S3Config s3Config; + private final DataMapper dataMapper; private final QueryConfig queryConfig; private final MariaDbMapper mariaDbMapper; private final SchemaService schemaService; @@ -45,11 +47,12 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe private final DataDatabaseSidecarGateway dataDatabaseSidecarGateway; @Autowired - public ViewServiceMariaDbImpl(S3Config s3Config, QueryConfig queryConfig, MariaDbMapper mariaDbMapper, - SchemaService schemaService, StorageService storageService, - MetadataMapper metadataMapper, + public ViewServiceMariaDbImpl(S3Config s3Config, DataMapper dataMapper, QueryConfig queryConfig, + MariaDbMapper mariaDbMapper, SchemaService schemaService, + StorageService storageService, MetadataMapper metadataMapper, DataDatabaseSidecarGateway dataDatabaseSidecarGateway) { this.s3Config = s3Config; + this.dataMapper = dataMapper; this.queryConfig = queryConfig; this.mariaDbMapper = mariaDbMapper; this.schemaService = schemaService; @@ -122,7 +125,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe statement2.setString(2, view.getInternalName()); final ResultSet resultSet2 = statement2.executeQuery(); while (resultSet2.next()) { - view = mariaDbMapper.resultSetToTable(resultSet2, view, queryConfig); + view = dataMapper.resultSetToTable(resultSet2, view, queryConfig); } connection.commit(); } catch (SQLException e) { @@ -152,7 +155,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe mariaDbMapper.selectDatasetRawQuery(view.getDatabase().getInternalName(), view.getInternalName(), mappedColumns, timestamp, size, page)) .executeQuery(); - queryResult = mariaDbMapper.resultListToQueryResultDto(mappedColumns, resultSet); + queryResult = dataMapper.resultListToQueryResultDto(mappedColumns, resultSet); queryResult.setId(view.getId()); connection.commit(); } catch (SQLException e) { diff --git a/helm/dbrepo/values.yaml b/helm/dbrepo/values.yaml index 15e6888d17c4df8e39de737cdd2ef6a946d3df1e..c9d8a260bf7b1c611e462c25f44f47ef7a8bc6ff 100644 --- a/helm/dbrepo/values.yaml +++ b/helm/dbrepo/values.yaml @@ -178,7 +178,7 @@ datadb: protocol: TCP sidecars: - name: sidecar - image: s210.dl.hpc.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/data-db-sidecar:1.4.4 imagePullPolicy: Always securityContext: runAsUser: 1001 @@ -338,6 +338,8 @@ uploadservice: tag: v1.12 securityContext: allowPrivilegeEscalation: false + runAsUser: 1000 + runAsGroup: 1000 runAsNonRoot: true seccompProfile: type: RuntimeDefault @@ -446,7 +448,7 @@ brokerservice: analyseservice: enabled: true image: - name: s210.dl.hpc.tuwien.ac.at/dbrepo/analyse-service:1.4.4 + name: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.4 pullPolicy: Always debug: false endpoint: http://analyse-service @@ -478,7 +480,7 @@ analyseservice: metadataservice: enabled: true image: - name: s210.dl.hpc.tuwien.ac.at/dbrepo/metadata-service:1.4.4 + name: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.4 pullPolicy: Always debug: false endpoint: http://metadata-service @@ -529,7 +531,7 @@ dataservice: enabled: true endpoint: http://data-service image: - name: s210.dl.hpc.tuwien.ac.at/dbrepo/data-service:1.4.4 + name: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.4 pullPolicy: Always debug: false grant: @@ -565,12 +567,12 @@ searchservice: enabled: true endpoint: http://search-service image: - name: s210.dl.hpc.tuwien.ac.at/dbrepo/search-service:1.4.4 + name: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.4 pullPolicy: Always debug: false init: image: - name: s210.dl.hpc.tuwien.ac.at/dbrepo/search-service-init:1.4.4 + name: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.4 pullPolicy: Always replicaCount: 2 @@ -617,7 +619,7 @@ storageservice: username: seaweedfsadmin password: seaweedfsadmin init: - image: s210.dl.hpc.tuwien.ac.at/dbrepo/storage-service-init:1.4.4 + image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.4.4 pullPolicy: Always ## @section User Interface @@ -646,7 +648,7 @@ storageservice: ui: enabled: true image: - name: s210.dl.hpc.tuwien.ac.at/dbrepo/ui:1.4.4 + name: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.4 pullPolicy: Always debug: false public: