Skip to content
Snippets Groups Projects
Unverified Commit 5d4abdef authored by Martin Weise's avatar Martin Weise
Browse files

Tons of new imporvements to search service and identifier service

parent 9bbc4d23
No related branches found
No related tags found
1 merge request!135Enable Elastic Search
Showing
with 122 additions and 41 deletions
......@@ -20,3 +20,4 @@ SHARED_FILESYSTEM=/tmp
LOG_LEVEL=trace # error, warning, info, debug, trace
DEFAULT_ROLES=ROLE_RESEARCHER
SUPERUSERS=user1,user2
ELASTIC_PASSWORD=elastic
\ No newline at end of file
......@@ -20,3 +20,4 @@ SHARED_FILESYSTEM=C:\tmp
LOG_LEVEL=trace # error, warning, info, debug, trace
DEFAULT_ROLES=ROLE_RESEARCHER
SUPERUSERS=user1,user2
ELASTIC_PASSWORD=elastic
\ No newline at end of file
......@@ -56,7 +56,7 @@ build-frontend:
build-clients:
bash ./.gitlab/swagger/generate.sh
tag: tag-identifier tag-container tag-database tag-discovery tag-gateway tag-query tag-table tag-analyse tag-authentication tag-metadata-db tag-ui tag-units tag-broker tag-ui-proxy tag-metadata
tag: tag-identifier tag-search tag-container tag-database tag-discovery tag-gateway tag-query tag-table tag-analyse tag-authentication tag-metadata-db tag-ui tag-units tag-broker tag-ui-proxy tag-metadata
tag-analyse:
docker tag fda-analyse-service:latest "dbrepo/analyse-service:${TAG}"
......@@ -103,7 +103,10 @@ tag-units:
tag-broker:
docker tag fda-broker-service:latest "dbrepo/broker-service:${TAG}"
release: build-docker tag release-identifier release-container release-database release-discovery release-gateway release-query release-table release-analyse release-authentication release-metadata-db release-ui release-units release-broker release-ui-proxy release-metadata
tag-search:
docker tag fda-search-service:latest "dbrepo/search-service:${TAG}"
release: build-docker tag release-identifier release-search release-container release-database release-discovery release-gateway release-query release-table release-analyse release-authentication release-metadata-db release-ui release-units release-broker release-ui-proxy release-metadata
release-analyse:
docker push "dbrepo/analyse-service:${TAG}"
......@@ -147,10 +150,13 @@ release-units:
release-broker:
docker push "dbrepo/broker-service:${TAG}"
release-search:
docker push "dbrepo/search-service:${TAG}"
release-metadata:
docker push "dbrepo/metadata-service:${TAG}"
pull: pull-identifier pull-container pull-database pull-discovery pull-gateway pull-query pull-table pull-analyse pull-authentication pull-metadata-db pull-ui pull-units pull-broker pull-ui-proxy pull-metadata
pull: pull-identifier pull-container pull-search pull-database pull-discovery pull-gateway pull-query pull-table pull-analyse pull-authentication pull-metadata-db pull-ui pull-units pull-broker pull-ui-proxy pull-metadata
pull-analyse:
docker pull "dbrepo/analyse-service:${TAG}"
......@@ -197,6 +203,9 @@ pull-broker:
pull-metadata:
docker pull "dbrepo/metadata-service:${TAG}"
pull-search:
docker pull "dbrepo/search-service:${TAG}"
test-backend: test-authentication-service test-container-service test-database-service test-discovery-service test-gateway-service test-query-service test-table-service test-identifier-service test-metadata-service test-semantics-service test-analyse-service
test-authentication-service: build-backend-metadata-db
......
......@@ -308,14 +308,10 @@ services:
restart: always
container_name: dbrepo-search-service
hostname: search-service
image: elasticsearch:7.13.4
command: [ "elasticsearch" ]
build: ./fda-search-service
image: fda-search-service
networks:
core:
environment:
- discovery.type=single-node
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- logger.level=WARN
depends_on:
fda-discovery-service:
condition: service_healthy
......
......@@ -39,8 +39,8 @@ eureka:
client.serviceUrl.defaultZone: http://discovery-service:9090/eureka/
fda:
elastic:
endpoint: "${SEARCH_ENDPOINT}"
username: "${SEARCH_USERNAME}"
password: "${SEARCH_PASSWORD}"
endpoint: search-service:9200
username: elastic
password: "${ELASTIC_PASSWORD}"
ready.path: /ready
gateway.endpoint: "${GATEWAY_ENDPOINT}"
\ No newline at end of file
......@@ -39,7 +39,7 @@ eureka:
client.serviceUrl.defaultZone: http://localhost:9090/eureka/
fda:
elastic:
endpoint: localhost:9200
endpoint: search-service:9200
username: elastic
password: elastic
ready.path: ./ready
......
......@@ -39,8 +39,8 @@ eureka:
client.serviceUrl.defaultZone: http://discovery-service:9090/eureka/
fda:
elastic:
endpoint: "${SEARCH_ENDPOINT}"
username: "${SEARCH_USERNAME}"
password: "${SEARCH_PASSWORD}"
endpoint: search-service:9200
username: elastic
password: "${ELASTIC_PASSWORD}"
ready.path: /ready
gateway.endpoint: http://gateway-service:9095
\ No newline at end of file
......@@ -25,6 +25,7 @@ public class ElasticsearchConfig {
public RestHighLevelClient client() {
final ClientConfiguration clientConfiguration = ClientConfiguration.builder()
.connectedTo(elasticEndpoint)
.withBasicAuth(elasticUsername, elasticPassword)
.build();
return RestClients.create(clientConfiguration)
.rest();
......
......@@ -12,6 +12,7 @@ import org.springframework.core.env.Environment;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import java.util.stream.Collectors;
......@@ -38,6 +39,7 @@ public class IndexConfig {
this.applicationEventPublisher = applicationEventPublisher;
}
@Transactional
@EventListener(ApplicationReadyEvent.class)
public void initIndex() {
log.debug("creating databaseindex");
......
package at.tuwien.config;
import at.tuwien.converters.IdentifierTypeConverter;
import org.springframework.context.annotation.Configuration;
import org.springframework.format.FormatterRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
@Configuration
public class WebConfig implements WebMvcConfigurer {
@Override
public void addFormatters(FormatterRegistry registry) {
registry.addConverter(new IdentifierTypeConverter());
}
}
package at.tuwien.converters;
import at.tuwien.api.identifier.IdentifierTypeDto;
import org.springframework.core.convert.converter.Converter;
public class IdentifierTypeConverter implements Converter<String, IdentifierTypeDto> {
@Override
public IdentifierTypeDto convert(String source) {
return IdentifierTypeDto.valueOf(source.toUpperCase());
}
}
package at.tuwien.endpoints;
import at.tuwien.api.database.query.QueryTypeDto;
import at.tuwien.api.identifier.IdentifierCreateDto;
import at.tuwien.api.identifier.IdentifierDto;
import at.tuwien.api.identifier.IdentifierTypeDto;
......@@ -48,11 +49,18 @@ public class IdentifierEndpoint extends AbstractEndpoint {
@Timed(value = "identifier.list", description = "Time needed to list the identifiers")
@Operation(summary = "Find identifiers")
public ResponseEntity<List<IdentifierDto>> list(@RequestParam(required = false) Long dbid,
@RequestParam(required = false) Long qid) {
log.debug("endpoint find identifiers, dbid={}, qid={}", dbid, qid);
@RequestParam(required = false) Long qid,
@RequestParam(required = false) IdentifierTypeDto type) {
log.debug("endpoint find identifiers, dbid={}, qid={}, type={}", dbid, qid, type);
final List<Identifier> identifiers = identifierService.findAll(dbid, qid);
final List<IdentifierDto> dto = identifiers.stream()
.map(identifierMapper::identifierToIdentifierDto)
.filter(i -> {
if (type != null) {
return i.getType().equals(type);
}
return true;
})
.collect(Collectors.toList());
log.info("Find identifiers resulted in {} identifiers", identifiers.size());
log.trace("endpoint find identifiers, list={}", dto);
......
......@@ -37,4 +37,7 @@ fda:
ready.path: /ready
gateway.endpoint: "${GATEWAY_ENDPOINT}"
website: "${WEBSITE}"
elastic.endpoint: search-service:9200
\ No newline at end of file
elastic:
endpoint: search-service:9200
username: elastic
password: "${ELASTIC_PASSWORD}"
\ No newline at end of file
......@@ -19,7 +19,7 @@ spring:
jdbc:
time_zone: UTC
application:
name: authentication-service
name: identifier-service
cloud:
loadbalancer.ribbon.enabled: false
server.port: 9096
......@@ -31,10 +31,13 @@ logging:
at.tuwien.auth.UserPermissionEvaluator: trace
org.springframework.web.servlet.mvc.method.annotation.ExceptionHandlerExceptionResolver: debug
eureka:
instance.hostname: authentication-service
instance.hostname: identifier-service
client.serviceUrl.defaultZone: http://localhost:9090/eureka/
fda:
ready.path: ./ready
website: http://localhost:3000
gateway.endpoint: http://localhost:9095
elastic.endpoint: localhost:9200
\ No newline at end of file
website: http://localhost:3000
elastic:
endpoint: search-service:9200
username: elastic
password: elastic
\ No newline at end of file
......@@ -37,4 +37,7 @@ fda:
ready.path: /ready
gateway.endpoint: "${GATEWAY_ENDPOINT}"
website: "${WEBSITE}"
elastic.endpoint: search-service:9200
\ No newline at end of file
elastic:
endpoint: search-service:9200
username: elastic
password: "${ELASTIC_PASSWORD}"
\ No newline at end of file
......@@ -15,10 +15,17 @@ public class ElasticsearchConfig {
@Value("${fda.elastic.endpoint}")
private String elasticEndpoint;
@Value("${fda.elastic.username}")
private String elasticUsername;
@Value("${fda.elastic.password}")
private String elasticPassword;
@Bean
public RestHighLevelClient client() {
ClientConfiguration clientConfiguration = ClientConfiguration.builder()
.connectedTo(elasticEndpoint)
.withBasicAuth(elasticUsername, elasticPassword)
.build();
return RestClients.create(clientConfiguration).rest();
......
package at.tuwien.config;
import at.tuwien.api.identifier.IdentifierDto;
import at.tuwien.mapper.IdentifierMapper;
import at.tuwien.repository.elastic.IdentifierIdxRepository;
import at.tuwien.repository.jpa.IdentifierRepository;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.core.env.Environment;
import org.springframework.core.env.Profiles;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import java.util.stream.Collectors;
@Component
@Log4j2
public class IndexInitializer {
private final Environment environment;
private final IdentifierMapper identifierMapper;
private final IdentifierRepository identifierRepository;
private final IdentifierIdxRepository identifierIdxRepository;
private final ElasticsearchOperations elasticsearchOperations;
@Autowired
public IndexInitializer(Environment environment, ElasticsearchOperations elasticsearchOperations) {
this.environment = environment;
public IndexInitializer(IdentifierMapper identifierMapper, IdentifierRepository identifierRepository,
IdentifierIdxRepository identifierIdxRepository,
ElasticsearchOperations elasticsearchOperations) {
this.identifierMapper = identifierMapper;
this.identifierRepository = identifierRepository;
this.identifierIdxRepository = identifierIdxRepository;
this.elasticsearchOperations = elasticsearchOperations;
}
@Transactional
@EventListener(ApplicationReadyEvent.class)
public void initIndex() {
if (environment.acceptsProfiles(Profiles.of("test-noelastic"))) {
return;
}
log.debug("creating identifierindex");
final IndexCoordinates identifierIndex = IndexCoordinates.of("identifierindex");
if (!elasticsearchOperations.indexOps(identifierIndex).exists()) {
elasticsearchOperations.indexOps(identifierIndex).create();
elasticsearchOperations.indexOps(identifierIndex).createMapping(IdentifierDto.class);
}
final List<IdentifierDto> identifiers = identifierRepository.findAll()
.stream()
.map(identifierMapper::identifierToIdentifierDto)
.collect(Collectors.toList());
log.debug("add {} identifiers to elastic search index", identifiers.size());
identifierIdxRepository.saveAll(identifiers);
}
}
package at.tuwien.service;
import at.tuwien.api.identifier.BibliographyTypeDto;
import at.tuwien.api.identifier.IdentifierCreateDto;
import at.tuwien.api.identifier.*;
import at.tuwien.ExportResource;
import at.tuwien.api.identifier.IdentifierDto;
import at.tuwien.api.identifier.VisibilityTypeDto;
import at.tuwien.entities.identifier.Identifier;
import at.tuwien.exception.*;
import org.springframework.core.io.InputStreamResource;
......
package at.tuwien.api.database.query;
import at.tuwien.api.identifier.IdentifierBriefDto;
import at.tuwien.api.identifier.IdentifierDto;
import at.tuwien.api.user.UserDto;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonIgnore;
......@@ -65,6 +67,8 @@ public class QueryBriefDto {
@Schema(example = "query")
private QueryTypeDto type;
private IdentifierBriefDto identifier;
@NotNull(message = "created timestamp is required")
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC")
private Instant created;
......
package at.tuwien.api.database.query;
import at.tuwien.api.identifier.IdentifierDto;
import at.tuwien.api.user.UserDto;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.fasterxml.jackson.annotation.JsonIgnore;
......@@ -55,18 +56,20 @@ public class QueryDto {
@Schema(example = "query")
private QueryTypeDto type;
private IdentifierDto identifier;
@NotBlank(message = "query hash is required")
@JsonProperty("query_hash")
@Parameter(example = "17e682f060b5f8e47ea04c5c4855908b0a5ad612022260fe50e11ecb0cc0ab76")
@Schema(example = "17e682f060b5f8e47ea04c5c4855908b0a5ad612022260fe50e11ecb0cc0ab76")
private String queryHash;
@NotNull
@JsonProperty("is_persisted")
@Parameter(example = "true")
@Schema(example = "true")
private Boolean isPersisted;
@JsonProperty("result_hash")
@Parameter(example = "17e682f060b5f8e47ea04c5c4855908b0a5ad612022260fe50e11ecb0cc0ab76")
@Schema(example = "17e682f060b5f8e47ea04c5c4855908b0a5ad612022260fe50e11ecb0cc0ab76")
private String resultHash;
@JsonProperty("result_number")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment