From 40ad9c5d47274a59f74a56d3eb17d6045ef67a61 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Wed, 7 Jun 2023 08:37:53 +0200
Subject: [PATCH] Indexes are constructed properly
---
dbrepo-database-service/pom.xml | 5 +++++
.../service/impl/MariaDbServiceImpl.java | 13 +++++++------
.../at/tuwien/api/database/DatabaseDto.java | 4 ++++
.../java/at/tuwien/api/database/ViewDto.java | 7 +++++++
.../tuwien/api/database/table/TableDto.java | 7 +++++++
.../api/database/table/columns/ColumnDto.java | 13 +++++++++++++
.../main/java/at/tuwien/api/user/UserDto.java | 2 ++
.../repository/mdb/TableColumnRepository.java | 2 +-
.../tuwien/service/impl/ViewServiceImpl.java | 10 ++++------
.../repository/mdb/TableColumnRepository.java | 2 +-
.../repository/mdb/TableColumnRepository.java | 2 +-
.../tuwien/service/impl/TableServiceImpl.java | 8 ++++----
.../tuwien/service/impl/UserServiceImpl.java | 19 ++++++++++++-------
13 files changed, 68 insertions(+), 26 deletions(-)
diff --git a/dbrepo-database-service/pom.xml b/dbrepo-database-service/pom.xml
index 8c0a29fdaf..dc1a4e4446 100644
--- a/dbrepo-database-service/pom.xml
+++ b/dbrepo-database-service/pom.xml
@@ -120,6 +120,11 @@
<artifactId>spring-data-opensearch-starter</artifactId>
<version>${opensearch-client.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.opensearch.client</groupId>
+ <artifactId>opensearch-java</artifactId>
+ <version>2.5.0</version>
+ </dependency>
<!-- Entity, API, QueryStore -->
<dependency>
<groupId>at.tuwien</groupId>
diff --git a/dbrepo-database-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java b/dbrepo-database-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java
index 93c3564be7..0725475a57 100644
--- a/dbrepo-database-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java
+++ b/dbrepo-database-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java
@@ -1,6 +1,7 @@
package at.tuwien.service.impl;
import at.tuwien.api.database.DatabaseCreateDto;
+import at.tuwien.api.database.DatabaseDto;
import at.tuwien.api.database.DatabaseModifyVisibilityDto;
import at.tuwien.api.database.DatabaseTransferDto;
import at.tuwien.entities.container.Container;
@@ -107,11 +108,10 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
}
/* save in metadata database */
databaseRepository.deleteById(databaseId);
- log.info("Deleted database with id {}", databaseId);
- log.trace("deleted database {}", database);
+ log.info("Deleted database with id {} in metadata database", databaseId);
// delete in database_index - elastic search
databaseIdxRepository.deleteById(databaseId);
- log.info("Deleted database in elastic search with id {}", databaseId);
+ log.info("Deleted database with id {} in open search database", databaseId);
}
@Override
@@ -154,10 +154,11 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
log.info("Created user {} on database with owner access", user.getUsername());
/* save in metadata database */
final Database entity = databaseRepository.save(database);
- log.info("Created database with id {}", entity.getId());
+ log.info("Created database with id {} in metadata database", entity.getId());
/* save in database_index - elastic search */
- databaseIdxRepository.save(databaseMapper.databaseToDatabaseDto(entity));
- log.info("Saved database in elastic search with id {}", entity.getId());
+ final DatabaseDto databaseDto = databaseMapper.databaseToDatabaseDto(entity);
+ databaseIdxRepository.save(databaseDto);
+ log.info("Created database with id {} in open search database", entity.getId());
return entity;
}
diff --git a/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/DatabaseDto.java b/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/DatabaseDto.java
index 24a7799130..0717ed8c48 100644
--- a/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/DatabaseDto.java
+++ b/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/DatabaseDto.java
@@ -14,6 +14,7 @@ import lombok.extern.jackson.Jacksonized;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import org.springframework.data.elasticsearch.annotations.Document;
+import org.springframework.data.elasticsearch.annotations.Field;
import java.time.Instant;
import java.util.List;
@@ -36,6 +37,7 @@ public class DatabaseDto {
private String name;
@NotBlank
+ @Field(name = "exchange_name")
@JsonProperty("exchange_name")
@Schema(example = "dbrepo/air_quality")
private String exchangeName;
@@ -43,6 +45,7 @@ public class DatabaseDto {
private IdentifierDto identifier;
@NotBlank
+ @Field(name = "internal_name")
@JsonProperty("internal_name")
@Schema(example = "weather_australia")
private String internalName;
@@ -54,6 +57,7 @@ public class DatabaseDto {
private List<ViewBriefDto> views;
+ @Field(name = "is_public")
@JsonProperty("is_public")
@Schema(example = "true")
private Boolean isPublic;
diff --git a/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/ViewDto.java b/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/ViewDto.java
index 8aeb060772..031aaadd0a 100644
--- a/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/ViewDto.java
+++ b/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/ViewDto.java
@@ -11,6 +11,7 @@ import lombok.extern.jackson.Jacksonized;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import org.springframework.data.elasticsearch.annotations.Document;
+import org.springframework.data.elasticsearch.annotations.Field;
import java.time.Instant;
import java.util.UUID;
@@ -29,6 +30,8 @@ public class ViewDto {
private Long id;
@NotNull
+ @Field(name = "database_id")
+ @JsonProperty("database_id")
private Long vdbid;
@NotNull
@@ -40,13 +43,16 @@ public class ViewDto {
private String name;
@NotBlank
+ @Field(name = "air_quality")
@Schema(example = "air_quality")
private String internalName;
+ @Field(name = "is_public")
@JsonProperty("is_public")
@Schema(example = "true")
private Boolean isPublic;
+ @Field(name = "initial_view")
@JsonProperty("initial_view")
@Schema(example = "true", description = "True if it is the default view for the database")
private Boolean isInitialView;
@@ -66,6 +72,7 @@ public class ViewDto {
@NotNull
private UserDto creator;
+ @Field(name = "last_modified")
@JsonProperty("last_modified")
@Schema(example = "2020-08-04 11:12:00")
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC")
diff --git a/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/table/TableDto.java b/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/table/TableDto.java
index 8696a09278..8df51261bd 100644
--- a/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/table/TableDto.java
+++ b/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/table/TableDto.java
@@ -12,6 +12,7 @@ import lombok.extern.jackson.Jacksonized;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import org.springframework.data.elasticsearch.annotations.Document;
+import org.springframework.data.elasticsearch.annotations.Field;
import java.time.Instant;
import java.util.List;
@@ -27,10 +28,12 @@ import java.util.List;
public class TableDto {
@NotNull
+ @Field(name = "container_id")
@JsonProperty("container_id")
private Long containerId;
@NotNull
+ @Field(name = "database_id")
@JsonProperty("database_id")
private Long databaseId;
@@ -42,6 +45,7 @@ public class TableDto {
private String name;
@NotBlank(message = "internalName is required")
+ @Field(name = "internal_name")
@JsonProperty("internal_name")
@Schema(example = "air_quality")
private String internalName;
@@ -53,11 +57,13 @@ public class TableDto {
private UserBriefDto owner;
@NotBlank(message = "queueName is required")
+ @Field(name = "queue_name")
@JsonProperty("queue_name")
@Schema(example = "dbrepo/air_quality/air_quality")
private String queueName;
@NotBlank(message = "routingKey is required")
+ @Field(name = "routing_key")
@JsonProperty("routing_key")
@Schema(example = "dbrepo/air_quality/air_quality/1")
private String routingKey;
@@ -67,6 +73,7 @@ public class TableDto {
private String description;
@NotNull(message = "isPublic is required")
+ @Field(name = "is_public")
@JsonProperty("is_public")
@Schema(example = "true")
private Boolean isPublic;
diff --git a/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java b/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java
index 68c7934e18..6f3b30b04d 100644
--- a/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java
+++ b/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java
@@ -12,6 +12,7 @@ import lombok.extern.jackson.Jacksonized;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import org.springframework.data.elasticsearch.annotations.Document;
+import org.springframework.data.elasticsearch.annotations.Field;
@Getter
@Setter
@@ -24,14 +25,17 @@ import org.springframework.data.elasticsearch.annotations.Document;
public class ColumnDto {
@NotNull
+ @Field(name = "container_id")
@JsonProperty("container_id")
private Long containerId;
@NotNull
+ @Field(name = "database_id")
@JsonProperty("database_id")
private Long databaseId;
@NotNull
+ @Field(name = "table_id")
@JsonProperty("table_id")
private Long tableId;
@@ -43,28 +47,34 @@ public class ColumnDto {
private String name;
@NotBlank
+ @Field(name = "internal_name")
@JsonProperty("internal_name")
@Schema(example = "mdb_date")
private String internalName;
+ @Field(name = "date_format")
@JsonProperty("date_format")
@org.springframework.data.annotation.Transient
private ImageDateDto dateFormat;
@NotNull
+ @Field(name = "auto_generated")
@JsonProperty("auto_generated")
@Schema(example = "false")
private Boolean autoGenerated;
@NotNull
+ @Field(name = "is_primary_key")
@JsonProperty("is_primary_key")
@Schema(example = "true")
private Boolean isPrimaryKey;
+ @Field(name = "index_length")
@JsonProperty("index_length")
private Integer indexLength;
@NotNull
+ @Field(name = "column_type")
@JsonProperty("column_type")
@Schema(example = "string")
private ColumnTypeDto columnType;
@@ -77,15 +87,18 @@ public class ColumnDto {
private UnitDto unit;
@NotNull
+ @Field(name = "is_public")
@JsonProperty("is_public")
@Schema(example = "true")
private Boolean isPublic;
@NotNull
+ @Field(name = "is_null_allowed")
@JsonProperty("is_null_allowed")
@Schema(example = "false")
private Boolean isNullAllowed;
+ @Field(name = "enum_values")
@JsonProperty("enum_values")
@Parameter(description = "enum values, only considered when type = ENUM")
private String[] enumValues;
diff --git a/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/user/UserDto.java b/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/user/UserDto.java
index 88a86abd65..2a8cfd4fe4 100644
--- a/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/user/UserDto.java
+++ b/dbrepo-metadata-db/api/src/main/java/at/tuwien/api/user/UserDto.java
@@ -8,6 +8,7 @@ import lombok.*;
import jakarta.validation.constraints.NotNull;
import lombok.extern.jackson.Jacksonized;
import org.springframework.data.elasticsearch.annotations.Document;
+import org.springframework.data.elasticsearch.annotations.Field;
import java.util.List;
import java.util.UUID;
@@ -66,6 +67,7 @@ public class UserDto {
private String email;
@NotNull
+ @Field(name = "email_verified")
@JsonProperty("email_verified")
@Schema(example = "true")
@org.springframework.data.annotation.Transient
diff --git a/dbrepo-query-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java b/dbrepo-query-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java
index ab94de26eb..dd34acefa6 100644
--- a/dbrepo-query-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java
+++ b/dbrepo-query-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java
@@ -8,7 +8,7 @@ import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository
-public interface TableColumnRepository extends JpaRepository<TableColumn, TableColumnKey> {
+public interface TableColumnRepository extends JpaRepository<TableColumn, Long> {
Optional<TableColumn> findByIdAndTidAndCdbid(Long id, Long tid, Long cdbid);
diff --git a/dbrepo-query-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java b/dbrepo-query-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java
index 7cb9a8636d..e259d39c46 100644
--- a/dbrepo-query-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java
+++ b/dbrepo-query-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java
@@ -105,10 +105,9 @@ public class ViewServiceImpl extends HibernateConnector implements ViewService {
}
/* delete in metadata database */
viewRepository.delete(view);
- log.info("Deleted view with id {}", view.getId());
- log.trace("deleted view {}", view);
+ log.info("Deleted view with id {} in metadata database", id);
viewIdxRepository.deleteById(id);
- log.info("Deleted view with id {} in elastic search", id);
+ log.info("Deleted view with id {} in open search database", id);
}
@Override
@@ -154,10 +153,9 @@ public class ViewServiceImpl extends HibernateConnector implements ViewService {
.columns(columns)
.build();
final View view = viewRepository.save(entity);
- log.info("Created view with id {}", view.getId());
- log.trace("created view {}", view);
+ log.info("Created view with id {} in metadata database", view.getId());
viewIdxRepository.save(viewMapper.viewToViewDto(view));
- log.info("Created view with id {} in elastic search", view.getId());
+ log.info("Created view with id {} in open search database", view.getId());
return view;
}
diff --git a/dbrepo-semantics-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java b/dbrepo-semantics-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java
index b52cc85d35..de41d04da2 100644
--- a/dbrepo-semantics-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java
+++ b/dbrepo-semantics-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java
@@ -6,6 +6,6 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
-public interface TableColumnRepository extends JpaRepository<TableColumn, TableColumnKey> {
+public interface TableColumnRepository extends JpaRepository<TableColumn, Long> {
}
diff --git a/dbrepo-table-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java b/dbrepo-table-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java
index b52cc85d35..de41d04da2 100644
--- a/dbrepo-table-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java
+++ b/dbrepo-table-service/services/src/main/java/at/tuwien/repository/mdb/TableColumnRepository.java
@@ -6,6 +6,6 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
@Repository
-public interface TableColumnRepository extends JpaRepository<TableColumn, TableColumnKey> {
+public interface TableColumnRepository extends JpaRepository<TableColumn, Long> {
}
diff --git a/dbrepo-table-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java b/dbrepo-table-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
index aada8f5a92..463dc7edd8 100644
--- a/dbrepo-table-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
+++ b/dbrepo-table-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
@@ -91,7 +91,7 @@ public class TableServiceImpl extends HibernateConnector implements TableService
tableRepository.delete(table);
log.info("Deleted table with id {} in metadata database", table.getId());
tableIdxRepository.delete(tableMapper.tableToTableDto(table));
- log.info("Deleted table with id {} in search service", table.getId());
+ log.info("Deleted table with id {} in open search database", table.getId());
}
@Override
@@ -190,13 +190,13 @@ public class TableServiceImpl extends HibernateConnector implements TableService
}
/* save in metadata database */
final Table table = tableRepository.save(entity);
- log.info("Created table with id {}", table.getId());
- log.trace("created table {}", table);
+ log.info("Created table with id {} in metadata database", table.getId());
/* save in database_index - elastic search */
tableIdxRepository.save(tableMapper.tableToTableDto(table));
+ log.info("Created table with id {} in open search database", table.getId());
/* save in column_index - elastic search */
tableColumnIdxRepository.saveAll(tableMapper.tableToTableDto(table).getColumns());
- log.info("Saved table with id {} in elastic search", table.getId());
+ log.info("Saved table columns with table id {} in open search database", table.getId());
return table;
}
diff --git a/dbrepo-user-service/services/src/main/java/at/tuwien/service/impl/UserServiceImpl.java b/dbrepo-user-service/services/src/main/java/at/tuwien/service/impl/UserServiceImpl.java
index a46bb17a13..b3e4a75d85 100644
--- a/dbrepo-user-service/services/src/main/java/at/tuwien/service/impl/UserServiceImpl.java
+++ b/dbrepo-user-service/services/src/main/java/at/tuwien/service/impl/UserServiceImpl.java
@@ -1,6 +1,7 @@
package at.tuwien.service.impl;
import at.tuwien.api.auth.SignupRequestDto;
+import at.tuwien.api.user.UserDto;
import at.tuwien.api.user.UserPasswordDto;
import at.tuwien.api.user.UserThemeSetDto;
import at.tuwien.api.user.UserUpdateDto;
@@ -8,8 +9,8 @@ import at.tuwien.entities.user.*;
import at.tuwien.exception.*;
import at.tuwien.mapper.UserMapper;
import at.tuwien.repository.mdb.CredentialRepository;
-import at.tuwien.repository.mdb.RoleMappingRepository;
import at.tuwien.repository.mdb.UserRepository;
+import at.tuwien.repository.sdb.UserIdxRepository;
import at.tuwien.service.UserAttributeService;
import at.tuwien.service.UserService;
import lombok.extern.log4j.Log4j2;
@@ -43,19 +44,18 @@ public class UserServiceImpl implements UserService {
private final UserMapper userMapper;
private final UserRepository userRepository;
+ private final UserIdxRepository userIdxRepository;
private final UserAttributeService userAttributeService;
private final CredentialRepository credentialRepository;
- private final RoleMappingRepository roleMappingRepository;
@Autowired
- public UserServiceImpl(UserMapper userMapper, UserRepository userRepository,
- UserAttributeService userAttributeService, CredentialRepository credentialRepository,
- RoleMappingRepository roleMappingRepository) {
+ public UserServiceImpl(UserMapper userMapper, UserRepository userRepository, UserIdxRepository userIdxRepository,
+ UserAttributeService userAttributeService, CredentialRepository credentialRepository) {
this.userMapper = userMapper;
this.userRepository = userRepository;
+ this.userIdxRepository = userIdxRepository;
this.userAttributeService = userAttributeService;
this.credentialRepository = credentialRepository;
- this.roleMappingRepository = roleMappingRepository;
}
@Override
@@ -97,11 +97,16 @@ public class UserServiceImpl implements UserService {
final UserAttribute userAttribute3 = userAttributeService.create(userMapper.tripleToUserAttribute(user.getId(),
"affiliation", ""));
credential.setUserId(user.getId());
+ /* save in metadata database */
credential = credentialRepository.save(credential);
user.setCredentials(List.of(credential));
user.setAttributes(List.of(userAttribute1, userAttribute2, userAttribute3));
user.setRoles(List.of(role));
- log.info("Created user with id {}", user.getId());
+ log.info("Created user with id {} in metadata database", user.getId());
+ /* save in open search database */
+ final UserDto userDto = userMapper.userToUserDto(user);
+ userIdxRepository.save(userDto);
+ log.info("Created user with id {} in open search database", user.getId());
return user;
}
--
GitLab