Skip to content
Snippets Groups Projects
Unverified Commit 40ad9c5d authored by Martin Weise's avatar Martin Weise
Browse files

Indexes are constructed properly

parent b4432333
Branches
Tags
6 merge requests!231CI: Remove build for log-service,!228Better error message handling in the frontend,!223Release of version 1.4.0,!177Updated keycloak realms to only allow creation of containers for developers,...,!176Hotfix views,!165Resolve "Migrate to OpenSearch"
Showing
with 68 additions and 26 deletions
...@@ -120,6 +120,11 @@ ...@@ -120,6 +120,11 @@
<artifactId>spring-data-opensearch-starter</artifactId> <artifactId>spring-data-opensearch-starter</artifactId>
<version>${opensearch-client.version}</version> <version>${opensearch-client.version}</version>
</dependency> </dependency>
<dependency>
<groupId>org.opensearch.client</groupId>
<artifactId>opensearch-java</artifactId>
<version>2.5.0</version>
</dependency>
<!-- Entity, API, QueryStore --> <!-- Entity, API, QueryStore -->
<dependency> <dependency>
<groupId>at.tuwien</groupId> <groupId>at.tuwien</groupId>
......
package at.tuwien.service.impl; package at.tuwien.service.impl;
import at.tuwien.api.database.DatabaseCreateDto; import at.tuwien.api.database.DatabaseCreateDto;
import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.DatabaseModifyVisibilityDto; import at.tuwien.api.database.DatabaseModifyVisibilityDto;
import at.tuwien.api.database.DatabaseTransferDto; import at.tuwien.api.database.DatabaseTransferDto;
import at.tuwien.entities.container.Container; import at.tuwien.entities.container.Container;
...@@ -107,11 +108,10 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe ...@@ -107,11 +108,10 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
} }
/* save in metadata database */ /* save in metadata database */
databaseRepository.deleteById(databaseId); databaseRepository.deleteById(databaseId);
log.info("Deleted database with id {}", databaseId); log.info("Deleted database with id {} in metadata database", databaseId);
log.trace("deleted database {}", database);
// delete in database_index - elastic search // delete in database_index - elastic search
databaseIdxRepository.deleteById(databaseId); databaseIdxRepository.deleteById(databaseId);
log.info("Deleted database in elastic search with id {}", databaseId); log.info("Deleted database with id {} in open search database", databaseId);
} }
@Override @Override
...@@ -154,10 +154,11 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe ...@@ -154,10 +154,11 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
log.info("Created user {} on database with owner access", user.getUsername()); log.info("Created user {} on database with owner access", user.getUsername());
/* save in metadata database */ /* save in metadata database */
final Database entity = databaseRepository.save(database); final Database entity = databaseRepository.save(database);
log.info("Created database with id {}", entity.getId()); log.info("Created database with id {} in metadata database", entity.getId());
/* save in database_index - elastic search */ /* save in database_index - elastic search */
databaseIdxRepository.save(databaseMapper.databaseToDatabaseDto(entity)); final DatabaseDto databaseDto = databaseMapper.databaseToDatabaseDto(entity);
log.info("Saved database in elastic search with id {}", entity.getId()); databaseIdxRepository.save(databaseDto);
log.info("Created database with id {} in open search database", entity.getId());
return entity; return entity;
} }
......
...@@ -14,6 +14,7 @@ import lombok.extern.jackson.Jacksonized; ...@@ -14,6 +14,7 @@ import lombok.extern.jackson.Jacksonized;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.NotNull;
import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import java.time.Instant; import java.time.Instant;
import java.util.List; import java.util.List;
...@@ -36,6 +37,7 @@ public class DatabaseDto { ...@@ -36,6 +37,7 @@ public class DatabaseDto {
private String name; private String name;
@NotBlank @NotBlank
@Field(name = "exchange_name")
@JsonProperty("exchange_name") @JsonProperty("exchange_name")
@Schema(example = "dbrepo/air_quality") @Schema(example = "dbrepo/air_quality")
private String exchangeName; private String exchangeName;
...@@ -43,6 +45,7 @@ public class DatabaseDto { ...@@ -43,6 +45,7 @@ public class DatabaseDto {
private IdentifierDto identifier; private IdentifierDto identifier;
@NotBlank @NotBlank
@Field(name = "internal_name")
@JsonProperty("internal_name") @JsonProperty("internal_name")
@Schema(example = "weather_australia") @Schema(example = "weather_australia")
private String internalName; private String internalName;
...@@ -54,6 +57,7 @@ public class DatabaseDto { ...@@ -54,6 +57,7 @@ public class DatabaseDto {
private List<ViewBriefDto> views; private List<ViewBriefDto> views;
@Field(name = "is_public")
@JsonProperty("is_public") @JsonProperty("is_public")
@Schema(example = "true") @Schema(example = "true")
private Boolean isPublic; private Boolean isPublic;
......
...@@ -11,6 +11,7 @@ import lombok.extern.jackson.Jacksonized; ...@@ -11,6 +11,7 @@ import lombok.extern.jackson.Jacksonized;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.NotNull;
import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import java.time.Instant; import java.time.Instant;
import java.util.UUID; import java.util.UUID;
...@@ -29,6 +30,8 @@ public class ViewDto { ...@@ -29,6 +30,8 @@ public class ViewDto {
private Long id; private Long id;
@NotNull @NotNull
@Field(name = "database_id")
@JsonProperty("database_id")
private Long vdbid; private Long vdbid;
@NotNull @NotNull
...@@ -40,13 +43,16 @@ public class ViewDto { ...@@ -40,13 +43,16 @@ public class ViewDto {
private String name; private String name;
@NotBlank @NotBlank
@Field(name = "air_quality")
@Schema(example = "air_quality") @Schema(example = "air_quality")
private String internalName; private String internalName;
@Field(name = "is_public")
@JsonProperty("is_public") @JsonProperty("is_public")
@Schema(example = "true") @Schema(example = "true")
private Boolean isPublic; private Boolean isPublic;
@Field(name = "initial_view")
@JsonProperty("initial_view") @JsonProperty("initial_view")
@Schema(example = "true", description = "True if it is the default view for the database") @Schema(example = "true", description = "True if it is the default view for the database")
private Boolean isInitialView; private Boolean isInitialView;
...@@ -66,6 +72,7 @@ public class ViewDto { ...@@ -66,6 +72,7 @@ public class ViewDto {
@NotNull @NotNull
private UserDto creator; private UserDto creator;
@Field(name = "last_modified")
@JsonProperty("last_modified") @JsonProperty("last_modified")
@Schema(example = "2020-08-04 11:12:00") @Schema(example = "2020-08-04 11:12:00")
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC") @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC")
......
...@@ -12,6 +12,7 @@ import lombok.extern.jackson.Jacksonized; ...@@ -12,6 +12,7 @@ import lombok.extern.jackson.Jacksonized;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.NotNull;
import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import java.time.Instant; import java.time.Instant;
import java.util.List; import java.util.List;
...@@ -27,10 +28,12 @@ import java.util.List; ...@@ -27,10 +28,12 @@ import java.util.List;
public class TableDto { public class TableDto {
@NotNull @NotNull
@Field(name = "container_id")
@JsonProperty("container_id") @JsonProperty("container_id")
private Long containerId; private Long containerId;
@NotNull @NotNull
@Field(name = "database_id")
@JsonProperty("database_id") @JsonProperty("database_id")
private Long databaseId; private Long databaseId;
...@@ -42,6 +45,7 @@ public class TableDto { ...@@ -42,6 +45,7 @@ public class TableDto {
private String name; private String name;
@NotBlank(message = "internalName is required") @NotBlank(message = "internalName is required")
@Field(name = "internal_name")
@JsonProperty("internal_name") @JsonProperty("internal_name")
@Schema(example = "air_quality") @Schema(example = "air_quality")
private String internalName; private String internalName;
...@@ -53,11 +57,13 @@ public class TableDto { ...@@ -53,11 +57,13 @@ public class TableDto {
private UserBriefDto owner; private UserBriefDto owner;
@NotBlank(message = "queueName is required") @NotBlank(message = "queueName is required")
@Field(name = "queue_name")
@JsonProperty("queue_name") @JsonProperty("queue_name")
@Schema(example = "dbrepo/air_quality/air_quality") @Schema(example = "dbrepo/air_quality/air_quality")
private String queueName; private String queueName;
@NotBlank(message = "routingKey is required") @NotBlank(message = "routingKey is required")
@Field(name = "routing_key")
@JsonProperty("routing_key") @JsonProperty("routing_key")
@Schema(example = "dbrepo/air_quality/air_quality/1") @Schema(example = "dbrepo/air_quality/air_quality/1")
private String routingKey; private String routingKey;
...@@ -67,6 +73,7 @@ public class TableDto { ...@@ -67,6 +73,7 @@ public class TableDto {
private String description; private String description;
@NotNull(message = "isPublic is required") @NotNull(message = "isPublic is required")
@Field(name = "is_public")
@JsonProperty("is_public") @JsonProperty("is_public")
@Schema(example = "true") @Schema(example = "true")
private Boolean isPublic; private Boolean isPublic;
......
...@@ -12,6 +12,7 @@ import lombok.extern.jackson.Jacksonized; ...@@ -12,6 +12,7 @@ import lombok.extern.jackson.Jacksonized;
import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.NotNull;
import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
@Getter @Getter
@Setter @Setter
...@@ -24,14 +25,17 @@ import org.springframework.data.elasticsearch.annotations.Document; ...@@ -24,14 +25,17 @@ import org.springframework.data.elasticsearch.annotations.Document;
public class ColumnDto { public class ColumnDto {
@NotNull @NotNull
@Field(name = "container_id")
@JsonProperty("container_id") @JsonProperty("container_id")
private Long containerId; private Long containerId;
@NotNull @NotNull
@Field(name = "database_id")
@JsonProperty("database_id") @JsonProperty("database_id")
private Long databaseId; private Long databaseId;
@NotNull @NotNull
@Field(name = "table_id")
@JsonProperty("table_id") @JsonProperty("table_id")
private Long tableId; private Long tableId;
...@@ -43,28 +47,34 @@ public class ColumnDto { ...@@ -43,28 +47,34 @@ public class ColumnDto {
private String name; private String name;
@NotBlank @NotBlank
@Field(name = "internal_name")
@JsonProperty("internal_name") @JsonProperty("internal_name")
@Schema(example = "mdb_date") @Schema(example = "mdb_date")
private String internalName; private String internalName;
@Field(name = "date_format")
@JsonProperty("date_format") @JsonProperty("date_format")
@org.springframework.data.annotation.Transient @org.springframework.data.annotation.Transient
private ImageDateDto dateFormat; private ImageDateDto dateFormat;
@NotNull @NotNull
@Field(name = "auto_generated")
@JsonProperty("auto_generated") @JsonProperty("auto_generated")
@Schema(example = "false") @Schema(example = "false")
private Boolean autoGenerated; private Boolean autoGenerated;
@NotNull @NotNull
@Field(name = "is_primary_key")
@JsonProperty("is_primary_key") @JsonProperty("is_primary_key")
@Schema(example = "true") @Schema(example = "true")
private Boolean isPrimaryKey; private Boolean isPrimaryKey;
@Field(name = "index_length")
@JsonProperty("index_length") @JsonProperty("index_length")
private Integer indexLength; private Integer indexLength;
@NotNull @NotNull
@Field(name = "column_type")
@JsonProperty("column_type") @JsonProperty("column_type")
@Schema(example = "string") @Schema(example = "string")
private ColumnTypeDto columnType; private ColumnTypeDto columnType;
...@@ -77,15 +87,18 @@ public class ColumnDto { ...@@ -77,15 +87,18 @@ public class ColumnDto {
private UnitDto unit; private UnitDto unit;
@NotNull @NotNull
@Field(name = "is_public")
@JsonProperty("is_public") @JsonProperty("is_public")
@Schema(example = "true") @Schema(example = "true")
private Boolean isPublic; private Boolean isPublic;
@NotNull @NotNull
@Field(name = "is_null_allowed")
@JsonProperty("is_null_allowed") @JsonProperty("is_null_allowed")
@Schema(example = "false") @Schema(example = "false")
private Boolean isNullAllowed; private Boolean isNullAllowed;
@Field(name = "enum_values")
@JsonProperty("enum_values") @JsonProperty("enum_values")
@Parameter(description = "enum values, only considered when type = ENUM") @Parameter(description = "enum values, only considered when type = ENUM")
private String[] enumValues; private String[] enumValues;
......
...@@ -8,6 +8,7 @@ import lombok.*; ...@@ -8,6 +8,7 @@ import lombok.*;
import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.NotNull;
import lombok.extern.jackson.Jacksonized; import lombok.extern.jackson.Jacksonized;
import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
...@@ -66,6 +67,7 @@ public class UserDto { ...@@ -66,6 +67,7 @@ public class UserDto {
private String email; private String email;
@NotNull @NotNull
@Field(name = "email_verified")
@JsonProperty("email_verified") @JsonProperty("email_verified")
@Schema(example = "true") @Schema(example = "true")
@org.springframework.data.annotation.Transient @org.springframework.data.annotation.Transient
......
...@@ -8,7 +8,7 @@ import org.springframework.stereotype.Repository; ...@@ -8,7 +8,7 @@ import org.springframework.stereotype.Repository;
import java.util.Optional; import java.util.Optional;
@Repository @Repository
public interface TableColumnRepository extends JpaRepository<TableColumn, TableColumnKey> { public interface TableColumnRepository extends JpaRepository<TableColumn, Long> {
Optional<TableColumn> findByIdAndTidAndCdbid(Long id, Long tid, Long cdbid); Optional<TableColumn> findByIdAndTidAndCdbid(Long id, Long tid, Long cdbid);
......
...@@ -105,10 +105,9 @@ public class ViewServiceImpl extends HibernateConnector implements ViewService { ...@@ -105,10 +105,9 @@ public class ViewServiceImpl extends HibernateConnector implements ViewService {
} }
/* delete in metadata database */ /* delete in metadata database */
viewRepository.delete(view); viewRepository.delete(view);
log.info("Deleted view with id {}", view.getId()); log.info("Deleted view with id {} in metadata database", id);
log.trace("deleted view {}", view);
viewIdxRepository.deleteById(id); viewIdxRepository.deleteById(id);
log.info("Deleted view with id {} in elastic search", id); log.info("Deleted view with id {} in open search database", id);
} }
@Override @Override
...@@ -154,10 +153,9 @@ public class ViewServiceImpl extends HibernateConnector implements ViewService { ...@@ -154,10 +153,9 @@ public class ViewServiceImpl extends HibernateConnector implements ViewService {
.columns(columns) .columns(columns)
.build(); .build();
final View view = viewRepository.save(entity); final View view = viewRepository.save(entity);
log.info("Created view with id {}", view.getId()); log.info("Created view with id {} in metadata database", view.getId());
log.trace("created view {}", view);
viewIdxRepository.save(viewMapper.viewToViewDto(view)); viewIdxRepository.save(viewMapper.viewToViewDto(view));
log.info("Created view with id {} in elastic search", view.getId()); log.info("Created view with id {} in open search database", view.getId());
return view; return view;
} }
......
...@@ -6,6 +6,6 @@ import org.springframework.data.jpa.repository.JpaRepository; ...@@ -6,6 +6,6 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
@Repository @Repository
public interface TableColumnRepository extends JpaRepository<TableColumn, TableColumnKey> { public interface TableColumnRepository extends JpaRepository<TableColumn, Long> {
} }
...@@ -6,6 +6,6 @@ import org.springframework.data.jpa.repository.JpaRepository; ...@@ -6,6 +6,6 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository; import org.springframework.stereotype.Repository;
@Repository @Repository
public interface TableColumnRepository extends JpaRepository<TableColumn, TableColumnKey> { public interface TableColumnRepository extends JpaRepository<TableColumn, Long> {
} }
...@@ -91,7 +91,7 @@ public class TableServiceImpl extends HibernateConnector implements TableService ...@@ -91,7 +91,7 @@ public class TableServiceImpl extends HibernateConnector implements TableService
tableRepository.delete(table); tableRepository.delete(table);
log.info("Deleted table with id {} in metadata database", table.getId()); log.info("Deleted table with id {} in metadata database", table.getId());
tableIdxRepository.delete(tableMapper.tableToTableDto(table)); tableIdxRepository.delete(tableMapper.tableToTableDto(table));
log.info("Deleted table with id {} in search service", table.getId()); log.info("Deleted table with id {} in open search database", table.getId());
} }
@Override @Override
...@@ -190,13 +190,13 @@ public class TableServiceImpl extends HibernateConnector implements TableService ...@@ -190,13 +190,13 @@ public class TableServiceImpl extends HibernateConnector implements TableService
} }
/* save in metadata database */ /* save in metadata database */
final Table table = tableRepository.save(entity); final Table table = tableRepository.save(entity);
log.info("Created table with id {}", table.getId()); log.info("Created table with id {} in metadata database", table.getId());
log.trace("created table {}", table);
/* save in database_index - elastic search */ /* save in database_index - elastic search */
tableIdxRepository.save(tableMapper.tableToTableDto(table)); tableIdxRepository.save(tableMapper.tableToTableDto(table));
log.info("Created table with id {} in open search database", table.getId());
/* save in column_index - elastic search */ /* save in column_index - elastic search */
tableColumnIdxRepository.saveAll(tableMapper.tableToTableDto(table).getColumns()); tableColumnIdxRepository.saveAll(tableMapper.tableToTableDto(table).getColumns());
log.info("Saved table with id {} in elastic search", table.getId()); log.info("Saved table columns with table id {} in open search database", table.getId());
return table; return table;
} }
......
package at.tuwien.service.impl; package at.tuwien.service.impl;
import at.tuwien.api.auth.SignupRequestDto; import at.tuwien.api.auth.SignupRequestDto;
import at.tuwien.api.user.UserDto;
import at.tuwien.api.user.UserPasswordDto; import at.tuwien.api.user.UserPasswordDto;
import at.tuwien.api.user.UserThemeSetDto; import at.tuwien.api.user.UserThemeSetDto;
import at.tuwien.api.user.UserUpdateDto; import at.tuwien.api.user.UserUpdateDto;
...@@ -8,8 +9,8 @@ import at.tuwien.entities.user.*; ...@@ -8,8 +9,8 @@ import at.tuwien.entities.user.*;
import at.tuwien.exception.*; import at.tuwien.exception.*;
import at.tuwien.mapper.UserMapper; import at.tuwien.mapper.UserMapper;
import at.tuwien.repository.mdb.CredentialRepository; import at.tuwien.repository.mdb.CredentialRepository;
import at.tuwien.repository.mdb.RoleMappingRepository;
import at.tuwien.repository.mdb.UserRepository; import at.tuwien.repository.mdb.UserRepository;
import at.tuwien.repository.sdb.UserIdxRepository;
import at.tuwien.service.UserAttributeService; import at.tuwien.service.UserAttributeService;
import at.tuwien.service.UserService; import at.tuwien.service.UserService;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
...@@ -43,19 +44,18 @@ public class UserServiceImpl implements UserService { ...@@ -43,19 +44,18 @@ public class UserServiceImpl implements UserService {
private final UserMapper userMapper; private final UserMapper userMapper;
private final UserRepository userRepository; private final UserRepository userRepository;
private final UserIdxRepository userIdxRepository;
private final UserAttributeService userAttributeService; private final UserAttributeService userAttributeService;
private final CredentialRepository credentialRepository; private final CredentialRepository credentialRepository;
private final RoleMappingRepository roleMappingRepository;
@Autowired @Autowired
public UserServiceImpl(UserMapper userMapper, UserRepository userRepository, public UserServiceImpl(UserMapper userMapper, UserRepository userRepository, UserIdxRepository userIdxRepository,
UserAttributeService userAttributeService, CredentialRepository credentialRepository, UserAttributeService userAttributeService, CredentialRepository credentialRepository) {
RoleMappingRepository roleMappingRepository) {
this.userMapper = userMapper; this.userMapper = userMapper;
this.userRepository = userRepository; this.userRepository = userRepository;
this.userIdxRepository = userIdxRepository;
this.userAttributeService = userAttributeService; this.userAttributeService = userAttributeService;
this.credentialRepository = credentialRepository; this.credentialRepository = credentialRepository;
this.roleMappingRepository = roleMappingRepository;
} }
@Override @Override
...@@ -97,11 +97,16 @@ public class UserServiceImpl implements UserService { ...@@ -97,11 +97,16 @@ public class UserServiceImpl implements UserService {
final UserAttribute userAttribute3 = userAttributeService.create(userMapper.tripleToUserAttribute(user.getId(), final UserAttribute userAttribute3 = userAttributeService.create(userMapper.tripleToUserAttribute(user.getId(),
"affiliation", "")); "affiliation", ""));
credential.setUserId(user.getId()); credential.setUserId(user.getId());
/* save in metadata database */
credential = credentialRepository.save(credential); credential = credentialRepository.save(credential);
user.setCredentials(List.of(credential)); user.setCredentials(List.of(credential));
user.setAttributes(List.of(userAttribute1, userAttribute2, userAttribute3)); user.setAttributes(List.of(userAttribute1, userAttribute2, userAttribute3));
user.setRoles(List.of(role)); user.setRoles(List.of(role));
log.info("Created user with id {}", user.getId()); log.info("Created user with id {} in metadata database", user.getId());
/* save in open search database */
final UserDto userDto = userMapper.userToUserDto(user);
userIdxRepository.save(userDto);
log.info("Created user with id {} in open search database", user.getId());
return user; return user;
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment