From 82e89eb1a3161d99427ea01b1a15c43d297a86ad Mon Sep 17 00:00:00 2001 From: Martin Weise <martin.weise@tuwien.ac.at> Date: Tue, 6 Jun 2023 10:58:28 +0200 Subject: [PATCH] WIP opensearch --- .env.unix.example | 3 +- README.md | 2 +- dbrepo-container-service/Dockerfile | 4 +- dbrepo-container-service/pom.xml | 17 ++++++ .../src/main/resources/application-local.yml | 10 ++-- .../src/main/resources/application.yml | 10 ++-- dbrepo-database-service/Dockerfile | 4 +- dbrepo-database-service/pom.xml | 7 +-- .../src/main/resources/application.yml | 6 +-- .../at/tuwien/config/ElasticsearchConfig.java | 45 ---------------- .../java/at/tuwien/config/IndexConfig.java | 15 ++---- .../at/tuwien/config/OpenSearchConfig.java | 35 ++++++++++++ dbrepo-identifier-service/pom.xml | 6 ++- .../src/main/resources/application-local.yml | 6 +-- .../src/main/resources/application.yml | 6 +-- .../endpoint/ActuatorComponentTest.java | 4 +- .../IdentifierEndpointIntegrationTest.java | 4 +- .../endpoint/IdentifierEndpointUnitTest.java | 5 +- .../endpoint/PersistenceEndpointUnitTest.java | 4 +- .../tuwien/endpoint/SwaggerComponentTest.java | 4 +- .../gateway/QueryServiceGatewayUnitTest.java | 4 +- .../DataCiteIdentifierServiceUnitTest.java | 5 +- .../IdentifierServiceIntegrationTest.java | 4 +- .../service/IdentifierServiceUnitTest.java | 5 +- .../tuwien/service/UserServiceUnitTest.java | 4 +- .../at/tuwien/config/ElasticsearchConfig.java | 45 ---------------- ...IndexInitializer.java => IndexConfig.java} | 14 ++--- .../at/tuwien/config/OpenSearchConfig.java | 35 ++++++++++++ dbrepo-metadata-db/pom.xml | 6 +++ .../main/java/at/tuwien/test/BaseTest.java | 2 +- dbrepo-metadata-service/pom.xml | 2 +- dbrepo-query-service/pom.xml | 8 +-- .../src/main/resources/application-local.yml | 6 +-- .../src/main/resources/application.yml | 6 +-- .../at/tuwien/config/ElasticsearchConfig.java | 45 ---------------- .../java/at/tuwien/config/IndexConfig.java | 17 ++---- .../at/tuwien/config/OpenSearchConfig.java | 35 ++++++++++++ dbrepo-search-db/README.md | 10 ++++ dbrepo-semantics-service/pom.xml | 6 +++ .../src/main/resources/application-local.yml | 6 +-- .../src/main/resources/application.yml | 6 +-- .../at/tuwien/config/ElasticsearchConfig.java | 45 ---------------- .../at/tuwien/config/OpenSearchConfig.java | 35 ++++++++++++ dbrepo-table-service/pom.xml | 6 ++- .../src/main/resources/application-local.yml | 6 +-- .../src/main/resources/application.yml | 6 +-- .../at/tuwien/config/ElasticsearchConfig.java | 45 ---------------- .../at/tuwien/config/OpenSearchConfig.java | 35 ++++++++++++ dbrepo-ui/.env.example | 4 +- dbrepo-ui/Dockerfile | 4 +- dbrepo-ui/config.js | 2 +- dbrepo-user-service/Dockerfile | 2 +- dbrepo-user-service/pom.xml | 8 +-- .../src/main/resources/application-local.yml | 6 +-- .../src/main/resources/application.yml | 6 +-- .../at/tuwien/config/ElasticsearchConfig.java | 45 ---------------- .../java/at/tuwien/config/IndexConfig.java | 53 +++++++++++++++++++ .../at/tuwien/config/OpenSearchConfig.java | 35 ++++++++++++ docker-compose.yml | 10 ++-- 59 files changed, 426 insertions(+), 395 deletions(-) delete mode 100644 dbrepo-database-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java create mode 100644 dbrepo-database-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java delete mode 100644 dbrepo-identifier-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java rename dbrepo-identifier-service/services/src/main/java/at/tuwien/config/{IndexInitializer.java => IndexConfig.java} (81%) create mode 100644 dbrepo-identifier-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java delete mode 100644 dbrepo-query-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java create mode 100644 dbrepo-query-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java create mode 100644 dbrepo-search-db/README.md delete mode 100644 dbrepo-semantics-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java create mode 100644 dbrepo-semantics-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java delete mode 100644 dbrepo-table-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java create mode 100644 dbrepo-table-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java delete mode 100644 dbrepo-user-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java create mode 100644 dbrepo-user-service/services/src/main/java/at/tuwien/config/IndexConfig.java create mode 100644 dbrepo-user-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java diff --git a/.env.unix.example b/.env.unix.example index 5e3cb074dc..e527bca539 100644 --- a/.env.unix.example +++ b/.env.unix.example @@ -4,7 +4,8 @@ JWT_ISSUER="http://localhost/realms/dbrepo" JWT_PUBKEY="MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB" JWT_CERT="MIICmzCCAYMCBgGG3GWyBTANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZkYnJlcG8wHhcNMjMwMzEzMTkxMzE3WhcNMzMwMzEzMTkxNDU3WjARMQ8wDQYDVQQDDAZkYnJlcG8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCqqcdDYFZZb28M0tEJzEP77FmD/Xqioyj9zWX6VwUSOMAgmMmn8eqs9hT9T0a+q4YTo9tUW1PNbUpwprA5b4Uk04DcIajxDVMUR/PjcHytmkqwVskq9AZW/Vngdoo+8tSbuIybwe/3Vwt266hbHpDcM97a+DXcYooRl7tQWCEX7RP27wQrMD9epDQ6IgKayZg9vC9/03dsIqwH9jXQRiZlFvwiEKhX2aY7lPGBaCK414JO00K/Z49iov9TRa/IYVbSt5qwgrx6DcqsBSPwOnI6A85UGfeUEZ/7coVJiL7RvBlsllapsL9eWTbQajVh94k9Ei3sibEPbtH+U2OAM78zAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAASnN1Cuif1sdfEK2kWAURSXGJCohCROLWdKFjaeHPRaEfpbFJsgxW0Yj3nwX5O3bUlOWoTyENwnXSsXMQsqnNi+At32CKaKO8+AkhAbgQL9F0B+KeJwmYv3cUj5N/LYkJjBvZBzUZ4Ugu5dcxH0k7AktLAIwimkyEnxTNolOA3UyrGGpREr8MCKWVr10RFuOpF/0CsJNNwbHXzalO9D756EUcRWZ9VSg6QVNso0YYRKTnILWDn9hcTRnqGy3SHo3anFTqQZ+BB57YbgFWy6udC0LYRB3zdp6zNti87eu/VEymiDY/mmo1AB8Tm0b6vxFz4AKcL3ax5qS6YnZ9efSzk=" SHARED_FILESYSTEM=/tmp -ELASTIC_PASSWORD=elastic +SEARCH_USERNAME=admin +SEARCH_PASSWORD=admin METADATA_DB=fda METADATA_USERNAME=root METADATA_PASSWORD=dbrepo diff --git a/README.md b/README.md index 1fde55325b..337f5d8129 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ $ docker-compose build --parallel A more detailed description on how to get started is available at our documentation -website: [https://dbrepo-docs.ossdip.at/getting-started/](https://dbrepo-docs.ossdip.at/getting-started/) +website: [https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//getting-started/](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//getting-started/) ### Run diff --git a/dbrepo-container-service/Dockerfile b/dbrepo-container-service/Dockerfile index 9e8c839383..9b27413bd7 100644 --- a/dbrepo-container-service/Dockerfile +++ b/dbrepo-container-service/Dockerfile @@ -30,8 +30,8 @@ ENV METADATA_USERNAME=root ENV METADATA_PASSWORD=dbrepo ENV BROKER_USERNAME=fda ENV BROKER_PASSWORD=fda -ENV ELASTIC_USERNAME=elastic -ENV ELASTIC_PASSWORD=elastic +ENV SEARCH_USERNAME=admin +ENV SEARCH_PASSWORD=admin ENV SHARED_FILESYSTEM=/tmp ENV USER_NETWORK=userdb ENV LOG_LEVEL=debug diff --git a/dbrepo-container-service/pom.xml b/dbrepo-container-service/pom.xml index c17167494d..8c4c65ef61 100644 --- a/dbrepo-container-service/pom.xml +++ b/dbrepo-container-service/pom.xml @@ -30,6 +30,7 @@ <jacoco.version>0.8.10</jacoco.version> <jwt.version>4.3.0</jwt.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch-client.version>1.1.0</opensearch-client.version> </properties> <dependencies> @@ -88,6 +89,16 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-jpa</artifactId> </dependency> + <dependency> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch</artifactId> + <version>${opensearch-client.version}</version> + </dependency> + <dependency> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch-client.version}</version> + </dependency> <!-- Entity and API --> <dependency> <groupId>at.tuwien</groupId> @@ -108,6 +119,12 @@ <version>${mariadb.version}</version> </dependency> <!-- Testing --> + <dependency> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-test-autoconfigure</artifactId> + <version>${opensearch-client.version}</version> + <scope>test</scope> + </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> diff --git a/dbrepo-container-service/rest-service/src/main/resources/application-local.yml b/dbrepo-container-service/rest-service/src/main/resources/application-local.yml index 9760a7ba9d..2af1c6bc21 100644 --- a/dbrepo-container-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-container-service/rest-service/src/main/resources/application-local.yml @@ -27,10 +27,6 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic - uris: http://localhost:9200 management.endpoints.web.exposure.include: health,info,prometheus springdoc.swagger-ui.enabled: true server: @@ -50,4 +46,8 @@ fda: public_key: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB client_secret: client-secret client_id: dbrepo-client - gateway.endpoint: http://localhost \ No newline at end of file + gateway.endpoint: http://localhost + opensearch: + username: admin + password: admin + uris: http://localhost:9200 \ No newline at end of file diff --git a/dbrepo-container-service/rest-service/src/main/resources/application.yml b/dbrepo-container-service/rest-service/src/main/resources/application.yml index 70619e94b9..907baa318f 100644 --- a/dbrepo-container-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-container-service/rest-service/src/main/resources/application.yml @@ -27,10 +27,6 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: "${ELASTIC_USERNAME}" - uris: http://search-db:9200 management.endpoints.web.exposure.include: health,info,prometheus springdoc.swagger-ui.enabled: true server: @@ -50,4 +46,8 @@ fda: public_key: "${JWT_PUBKEY}" client_secret: "${DBREPO_CLIENT_SECRET}" client_id: "${CLIENT_ID}" - gateway.endpoint: http://gateway-service \ No newline at end of file + gateway.endpoint: http://gateway-service + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" + uris: http://search-db:9200 \ No newline at end of file diff --git a/dbrepo-database-service/Dockerfile b/dbrepo-database-service/Dockerfile index bf79827ee7..c634dc3956 100644 --- a/dbrepo-database-service/Dockerfile +++ b/dbrepo-database-service/Dockerfile @@ -30,8 +30,8 @@ ENV METADATA_USERNAME=root ENV METADATA_PASSWORD=dbrepo ENV BROKER_USERNAME=fda ENV BROKER_PASSWORD=fda -ENV ELASTIC_USERNAME=elastic -ENV ELASTIC_PASSWORD=elastic +ENV SEARCH_USERNAME=admin +ENV SEARCH_PASSWORD=admin ENV GATEWAY_ENDPOINT=http://gateway-service ENV LOG_LEVEL=debug ENV CLIENT_ID="dbrepo-client" diff --git a/dbrepo-database-service/pom.xml b/dbrepo-database-service/pom.xml index d694d035a9..021bcf9c1a 100644 --- a/dbrepo-database-service/pom.xml +++ b/dbrepo-database-service/pom.xml @@ -31,6 +31,7 @@ <c3p0.version>0.9.5.5</c3p0.version> <c3p0-hibernate.version>6.2.2.Final</c3p0-hibernate.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -69,9 +70,9 @@ </dependency> <!-- elasticsearch --> <dependency> - <groupId>co.elastic.clients</groupId> - <artifactId>elasticsearch-java</artifactId> - <version>${elasticsearch-client.version}</version> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> </dependency> <!-- AMQP --> <dependency> diff --git a/dbrepo-database-service/rest-service/src/main/resources/application.yml b/dbrepo-database-service/rest-service/src/main/resources/application.yml index ba7ad2434e..3202f76b38 100644 --- a/dbrepo-database-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-database-service/rest-service/src/main/resources/application.yml @@ -27,9 +27,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: "${ELASTIC_USERNAME}" + opensearch: + password: "${SEARCH_PASSWORD}" + username: "${SEARCH_USERNAME}" uris: http://search-db:9200 management.endpoints.web.exposure.include: health,info,prometheus server: diff --git a/dbrepo-database-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-database-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c..0000000000 --- a/dbrepo-database-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-database-service/services/src/main/java/at/tuwien/config/IndexConfig.java b/dbrepo-database-service/services/src/main/java/at/tuwien/config/IndexConfig.java index dc0e46355d..69e950bac1 100644 --- a/dbrepo-database-service/services/src/main/java/at/tuwien/config/IndexConfig.java +++ b/dbrepo-database-service/services/src/main/java/at/tuwien/config/IndexConfig.java @@ -21,38 +21,33 @@ import java.util.stream.Collectors; @Component public class IndexConfig { - private final Environment environment; private final DatabaseMapper databaseMapper; private final DatabaseRepository databaseRepository; private final DatabaseIdxRepository databaseIdxRepository; private final ElasticsearchOperations elasticsearchOperations; - private final ApplicationEventPublisher applicationEventPublisher; - public IndexConfig(Environment environment, DatabaseMapper databaseMapper, DatabaseRepository databaseRepository, - DatabaseIdxRepository databaseIdxRepository, ElasticsearchOperations elasticsearchOperations, - ApplicationEventPublisher applicationEventPublisher) { - this.environment = environment; + public IndexConfig(DatabaseMapper databaseMapper, DatabaseRepository databaseRepository, + DatabaseIdxRepository databaseIdxRepository, ElasticsearchOperations elasticsearchOperations) { this.databaseMapper = databaseMapper; this.databaseRepository = databaseRepository; this.databaseIdxRepository = databaseIdxRepository; this.elasticsearchOperations = elasticsearchOperations; - this.applicationEventPublisher = applicationEventPublisher; } @Transactional @EventListener(ApplicationReadyEvent.class) public void initIndex() { - log.debug("creating databaseindex"); - final IndexCoordinates databaseIndex = IndexCoordinates.of("databaseindex"); + final IndexCoordinates databaseIndex = IndexCoordinates.of("database"); if (!elasticsearchOperations.indexOps(databaseIndex).exists()) { elasticsearchOperations.indexOps(databaseIndex).create(); elasticsearchOperations.indexOps(databaseIndex).createMapping(DatabaseDto.class); + log.info("Created identifier index"); } final List<DatabaseDto> databases = databaseRepository.findAll() .stream() .map(databaseMapper::databaseToDatabaseDto) .collect(Collectors.toList()); - log.debug("add {} databases to elastic search index", databases.size()); + log.info("Added {} databases to OpenSearch index", databases.size()); databaseIdxRepository.saveAll(databases); } diff --git a/dbrepo-database-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-database-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000..27d815cc6f --- /dev/null +++ b/dbrepo-database-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/dbrepo-identifier-service/pom.xml b/dbrepo-identifier-service/pom.xml index 07fe813de2..ef0bed4d9d 100644 --- a/dbrepo-identifier-service/pom.xml +++ b/dbrepo-identifier-service/pom.xml @@ -35,6 +35,7 @@ <super-csv.version>2.4.0</super-csv.version> <jsql.version>4.6</jsql.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -98,8 +99,9 @@ <version>${mariadb.version}</version> </dependency> <dependency> - <groupId>org.springframework.data</groupId> - <artifactId>spring-data-elasticsearch</artifactId> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> </dependency> <!-- Docker --> <dependency> diff --git a/dbrepo-identifier-service/rest-service/src/main/resources/application-local.yml b/dbrepo-identifier-service/rest-service/src/main/resources/application-local.yml index 6800891595..15a36b4bea 100644 --- a/dbrepo-identifier-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-identifier-service/rest-service/src/main/resources/application-local.yml @@ -27,9 +27,9 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic + opensearch: + username: admin + password: admin uris: http://localhost:9200 management.endpoints.web.exposure.include: health,info,prometheus server: diff --git a/dbrepo-identifier-service/rest-service/src/main/resources/application.yml b/dbrepo-identifier-service/rest-service/src/main/resources/application.yml index 0ec8f69c7b..23fcfe2727 100644 --- a/dbrepo-identifier-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-identifier-service/rest-service/src/main/resources/application.yml @@ -27,9 +27,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: elastic + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" uris: http://search-db:9200 management.endpoints.web.exposure.include: health,info,prometheus server: diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/ActuatorComponentTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/ActuatorComponentTest.java index 3677c7f045..2f817c1ec8 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/ActuatorComponentTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/ActuatorComponentTest.java @@ -1,7 +1,7 @@ package at.tuwien.endpoint; import at.tuwien.BaseUnitTest; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import lombok.extern.log4j.Log4j2; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -26,7 +26,7 @@ public class ActuatorComponentTest extends BaseUnitTest { private MockMvc mockMvc; @MockBean - private IndexInitializer indexConfig; + private IndexConfig indexConfig; @Test public void actuatorInfo_succeeds() throws Exception { diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointIntegrationTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointIntegrationTest.java index a2ea25e542..ddd2f989f2 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointIntegrationTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointIntegrationTest.java @@ -3,7 +3,7 @@ package at.tuwien.endpoint; import at.tuwien.BaseUnitTest; import at.tuwien.api.identifier.IdentifierDto; import at.tuwien.api.identifier.IdentifierTypeDto; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.config.ReadyConfig; import at.tuwien.endpoints.IdentifierEndpoint; import at.tuwien.exception.NotAllowedException; @@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.*; public class IdentifierEndpointIntegrationTest extends BaseUnitTest { @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private ReadyConfig readyConfig; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointUnitTest.java index 1be9690cb0..f94307ae80 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointUnitTest.java @@ -5,7 +5,7 @@ import at.tuwien.api.identifier.IdentifierCreateDto; import at.tuwien.api.identifier.IdentifierDto; import at.tuwien.api.identifier.IdentifierTypeDto; import at.tuwien.config.EndpointConfig; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.config.ReadyConfig; import at.tuwien.endpoints.IdentifierEndpoint; import at.tuwien.endpoints.PersistenceEndpoint; @@ -29,7 +29,6 @@ import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.AccessDeniedException; -import org.springframework.security.authentication.AuthenticationCredentialsNotFoundException; import org.springframework.security.test.context.support.WithAnonymousUser; import org.springframework.security.test.context.support.WithMockUser; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -54,7 +53,7 @@ public class IdentifierEndpointUnitTest extends BaseUnitTest { private ReadyConfig readyConfig; @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private IdentifierService identifierService; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/PersistenceEndpointUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/PersistenceEndpointUnitTest.java index d3c64bd929..7e9c11f73d 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/PersistenceEndpointUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/PersistenceEndpointUnitTest.java @@ -5,7 +5,7 @@ import at.tuwien.api.identifier.BibliographyTypeDto; import at.tuwien.api.identifier.CreatorDto; import at.tuwien.api.identifier.IdentifierDto; import at.tuwien.api.identifier.IdentifierUpdateDto; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.config.ReadyConfig; import at.tuwien.endpoints.PersistenceEndpoint; import at.tuwien.entities.identifier.Identifier; @@ -54,7 +54,7 @@ public class PersistenceEndpointUnitTest extends BaseUnitTest { private ReadyConfig readyConfig; @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private AccessService accessService; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/SwaggerComponentTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/SwaggerComponentTest.java index 0caa396455..47a1359841 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/SwaggerComponentTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/SwaggerComponentTest.java @@ -1,7 +1,7 @@ package at.tuwien.endpoint; import at.tuwien.BaseUnitTest; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import lombok.extern.log4j.Log4j2; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -26,7 +26,7 @@ public class SwaggerComponentTest extends BaseUnitTest { private MockMvc mockMvc; @MockBean - private IndexInitializer indexConfig; + private IndexConfig indexConfig; @Test public void swaggerUi_succeeds() throws Exception { diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/gateway/QueryServiceGatewayUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/gateway/QueryServiceGatewayUnitTest.java index 1a68af0041..6a39ebb85b 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/gateway/QueryServiceGatewayUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/gateway/QueryServiceGatewayUnitTest.java @@ -2,7 +2,7 @@ package at.tuwien.gateway; import at.tuwien.BaseUnitTest; import at.tuwien.api.database.query.QueryDto; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.config.ReadyConfig; import at.tuwien.exception.QueryNotFoundException; import at.tuwien.exception.RemoteUnavailableException; @@ -36,7 +36,7 @@ import static org.mockito.Mockito.*; public class QueryServiceGatewayUnitTest extends BaseUnitTest { @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private ReadyConfig readyConfig; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/DataCiteIdentifierServiceUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/DataCiteIdentifierServiceUnitTest.java index 1aad8f0b61..4bc2800a8f 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/DataCiteIdentifierServiceUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/DataCiteIdentifierServiceUnitTest.java @@ -8,7 +8,7 @@ import at.tuwien.api.identifier.IdentifierCreateDto; import at.tuwien.api.identifier.IdentifierUpdateDto; import at.tuwien.config.DataCiteConfig; import at.tuwien.config.EndpointConfig; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.entities.identifier.Identifier; import at.tuwien.exception.*; import at.tuwien.repository.jpa.*; @@ -29,7 +29,6 @@ import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; @@ -47,7 +46,7 @@ import static org.mockito.Mockito.when; public class DataCiteIdentifierServiceUnitTest extends BaseUnitTest { @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean(answer = Answers.RETURNS_MOCKS) private DataCiteConfig dataCiteConfig; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceIntegrationTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceIntegrationTest.java index 27bdb8c69b..8b826cb105 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceIntegrationTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceIntegrationTest.java @@ -3,7 +3,7 @@ package at.tuwien.service; import at.tuwien.BaseUnitTest; import at.tuwien.api.database.query.QueryDto; import at.tuwien.api.identifier.IdentifierDto; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.entities.identifier.Identifier; import at.tuwien.entities.identifier.RelatedIdentifier; import at.tuwien.exception.*; @@ -39,7 +39,7 @@ import static org.mockito.Mockito.when; public class IdentifierServiceIntegrationTest extends BaseUnitTest { @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private IdentifierIdxRepository identifierIdxRepository; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceUnitTest.java index d7e8b28963..b44d0282f5 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceUnitTest.java @@ -3,8 +3,7 @@ package at.tuwien.service; import at.tuwien.BaseUnitTest; import at.tuwien.api.database.query.QueryDto; import at.tuwien.api.identifier.IdentifierDto; -import at.tuwien.api.identifier.IdentifierUpdateDto; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.entities.identifier.Identifier; import at.tuwien.entities.identifier.IdentifierType; import at.tuwien.exception.*; @@ -39,7 +38,7 @@ public class IdentifierServiceUnitTest extends BaseUnitTest { private IdentifierRepository identifierRepository; @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private IdentifierIdxRepository identifierIdxRepository; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java index da772b10e1..baab96d40c 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java @@ -1,7 +1,7 @@ package at.tuwien.service; import at.tuwien.BaseUnitTest; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.entities.user.User; import at.tuwien.exception.*; import at.tuwien.repository.jpa.UserRepository; @@ -23,7 +23,7 @@ import static org.mockito.Mockito.when; public class UserServiceUnitTest extends BaseUnitTest { @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private UserRepository userRepository; diff --git a/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c..0000000000 --- a/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexInitializer.java b/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexConfig.java similarity index 81% rename from dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexInitializer.java rename to dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexConfig.java index 0592364d2d..2f0a40e3b4 100644 --- a/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexInitializer.java +++ b/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexConfig.java @@ -18,7 +18,7 @@ import java.util.stream.Collectors; @Component @Log4j2 -public class IndexInitializer { +public class IndexConfig { private final IdentifierMapper identifierMapper; private final IdentifierRepository identifierRepository; @@ -26,9 +26,9 @@ public class IndexInitializer { private final ElasticsearchOperations elasticsearchOperations; @Autowired - public IndexInitializer(IdentifierMapper identifierMapper, IdentifierRepository identifierRepository, - IdentifierIdxRepository identifierIdxRepository, - ElasticsearchOperations elasticsearchOperations) { + public IndexConfig(IdentifierMapper identifierMapper, IdentifierRepository identifierRepository, + IdentifierIdxRepository identifierIdxRepository, + ElasticsearchOperations elasticsearchOperations) { this.identifierMapper = identifierMapper; this.identifierRepository = identifierRepository; this.identifierIdxRepository = identifierIdxRepository; @@ -38,17 +38,17 @@ public class IndexInitializer { @Transactional @EventListener(ApplicationReadyEvent.class) public void initIndex() { - log.debug("creating identifierindex"); - final IndexCoordinates identifierIndex = IndexCoordinates.of("identifierindex"); + final IndexCoordinates identifierIndex = IndexCoordinates.of("identifier"); if (!elasticsearchOperations.indexOps(identifierIndex).exists()) { elasticsearchOperations.indexOps(identifierIndex).create(); elasticsearchOperations.indexOps(identifierIndex).createMapping(IdentifierDto.class); + log.info("Created identifier index"); } final List<IdentifierDto> identifiers = identifierRepository.findAll() .stream() .map(identifierMapper::identifierToIdentifierDto) .collect(Collectors.toList()); - log.debug("add {} identifiers to elastic search index", identifiers.size()); + log.info("Add {} identifiers to OpenSearch index", identifiers.size()); identifierIdxRepository.saveAll(identifiers); } } diff --git a/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000..27d815cc6f --- /dev/null +++ b/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/dbrepo-metadata-db/pom.xml b/dbrepo-metadata-db/pom.xml index cccf8f2178..284747e3d7 100644 --- a/dbrepo-metadata-db/pom.xml +++ b/dbrepo-metadata-db/pom.xml @@ -32,6 +32,7 @@ <jackson-datatype.version>2.15.0</jackson-datatype.version> <commons.version>2.11.0</commons.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -52,6 +53,11 @@ <groupId>org.springframework.data</groupId> <artifactId>spring-data-elasticsearch</artifactId> </dependency> + <dependency> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> + </dependency> <!-- Mapping --> <dependency> <groupId>com.fasterxml.jackson.datatype</groupId> diff --git a/dbrepo-metadata-db/test/src/main/java/at/tuwien/test/BaseTest.java b/dbrepo-metadata-db/test/src/main/java/at/tuwien/test/BaseTest.java index 22e0c0c39a..c9874c5799 100644 --- a/dbrepo-metadata-db/test/src/main/java/at/tuwien/test/BaseTest.java +++ b/dbrepo-metadata-db/test/src/main/java/at/tuwien/test/BaseTest.java @@ -1118,7 +1118,7 @@ public abstract class BaseTest { public final static String CONTAINER_ELASTIC_HASH = "deadbeef"; public final static Instant CONTAINER_ELASTIC_CREATED = Instant.ofEpochSecond(1677399721) /* 2023-02-26 08:22:01 (UTC) */; public final static String[] CONTAINER_ELASTIC_ENV = new String[]{"discovery.type=single-node", "ES_JAVA_OPTS=-Xms2g -Xmx2g", - "logger.level=WARN", "bootstrap.memory_lock=true", "xpack.security.enabled=true", "ELASTIC_PASSWORD=elastic"}; + "logger.level=WARN", "bootstrap.memory_lock=true", "xpack.security.enabled=true", "SEARCH_PASSWORD=admin"}; public final static Container CONTAINER_ELASTIC = Container.builder() .id(CONTAINER_ELASTIC_ID) diff --git a/dbrepo-metadata-service/pom.xml b/dbrepo-metadata-service/pom.xml index 261784bc3f..beddd0a5da 100644 --- a/dbrepo-metadata-service/pom.xml +++ b/dbrepo-metadata-service/pom.xml @@ -14,7 +14,7 @@ <name>dbrepo-metadata-service</name> <description>Service that manages the metadata</description> - <url>https://dbrepo-docs.ossdip.at</url> + <url>https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/</url> <developers> <developer> <name>Martin Weise</name> diff --git a/dbrepo-query-service/pom.xml b/dbrepo-query-service/pom.xml index 095adfeb17..9274612bde 100644 --- a/dbrepo-query-service/pom.xml +++ b/dbrepo-query-service/pom.xml @@ -17,7 +17,7 @@ also allows for view-only (possibly paginated and versioned) query execution to the raw data and consumes messages in the message queue from the Broker Service. </description> - <url>https://dbrepo-docs.ossdip.at</url> + <url>https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/</url> <developers> <developer> <name>Martin Weise</name> @@ -51,6 +51,7 @@ <c3p0.version>0.9.5.5</c3p0.version> <c3p0-hibernate.version>6.2.2.Final</c3p0-hibernate.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -121,8 +122,9 @@ <version>${mariadb.version}</version> </dependency> <dependency> - <groupId>org.springframework.data</groupId> - <artifactId>spring-data-elasticsearch</artifactId> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> </dependency> <!-- AMPQ --> <dependency> diff --git a/dbrepo-query-service/rest-service/src/main/resources/application-local.yml b/dbrepo-query-service/rest-service/src/main/resources/application-local.yml index 32b8a33c40..ee811a1e25 100644 --- a/dbrepo-query-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-query-service/rest-service/src/main/resources/application-local.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic + opensearch: + username: admin + password: admin uris: http://localhost:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-query-service/rest-service/src/main/resources/application.yml b/dbrepo-query-service/rest-service/src/main/resources/application.yml index db37d874ec..5caaab1e9b 100644 --- a/dbrepo-query-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-query-service/rest-service/src/main/resources/application.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: elastic + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" uris: http://search-db:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-query-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-query-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c..0000000000 --- a/dbrepo-query-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-query-service/services/src/main/java/at/tuwien/config/IndexConfig.java b/dbrepo-query-service/services/src/main/java/at/tuwien/config/IndexConfig.java index eeb61eac82..5b9b65d3dd 100644 --- a/dbrepo-query-service/services/src/main/java/at/tuwien/config/IndexConfig.java +++ b/dbrepo-query-service/services/src/main/java/at/tuwien/config/IndexConfig.java @@ -8,8 +8,6 @@ import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; -import org.springframework.core.env.Environment; -import org.springframework.core.env.Profiles; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.stereotype.Component; @@ -23,16 +21,14 @@ import java.util.stream.Collectors; public class IndexConfig { private final ViewMapper viewMapper; - private final Environment environment; private final ViewRepository viewRepository; private final ViewIdxRepository viewIdxRepository; private final ElasticsearchOperations elasticsearchOperations; @Autowired - public IndexConfig(ViewMapper viewMapper, Environment environment, ViewRepository viewRepository, - ViewIdxRepository viewIdxRepository, ElasticsearchOperations elasticsearchOperations) { + public IndexConfig(ViewMapper viewMapper, ViewRepository viewRepository, ViewIdxRepository viewIdxRepository, + ElasticsearchOperations elasticsearchOperations) { this.viewMapper = viewMapper; - this.environment = environment; this.viewRepository = viewRepository; this.viewIdxRepository = viewIdxRepository; this.elasticsearchOperations = elasticsearchOperations; @@ -41,21 +37,18 @@ public class IndexConfig { @Transactional @EventListener(ApplicationReadyEvent.class) public void initIndex() { - if (environment.acceptsProfiles(Profiles.of("test-noelastic"))) { - return; - } - log.debug("creating viewindex"); - final IndexCoordinates viewIndex = IndexCoordinates.of("viewindex"); + final IndexCoordinates viewIndex = IndexCoordinates.of("view"); if (!elasticsearchOperations.indexOps(viewIndex).exists()) { elasticsearchOperations.indexOps(viewIndex).create(); elasticsearchOperations.indexOps(viewIndex).createMapping(ViewDto.class); + log.info("Created identifier index"); } /* pre-fill */ final List<ViewDto> views = viewRepository.findAll() .stream() .map(viewMapper::viewToViewDto) .collect(Collectors.toList()); - log.debug("add {} views to elastic search index", views.size()); + log.info("Added {} views to OpenSearch index", views.size()); viewIdxRepository.saveAll(views); } } diff --git a/dbrepo-query-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-query-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000..27d815cc6f --- /dev/null +++ b/dbrepo-query-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/dbrepo-search-db/README.md b/dbrepo-search-db/README.md new file mode 100644 index 0000000000..541ebd8597 --- /dev/null +++ b/dbrepo-search-db/README.md @@ -0,0 +1,10 @@ + +# Search Database + +## How to run + +Increase memory for Docker to at least 4GB: + +```console +sudo sysctl -w vm.max_map_count=262144 +``` \ No newline at end of file diff --git a/dbrepo-semantics-service/pom.xml b/dbrepo-semantics-service/pom.xml index 3eed3ff468..08ba51d910 100644 --- a/dbrepo-semantics-service/pom.xml +++ b/dbrepo-semantics-service/pom.xml @@ -36,6 +36,7 @@ <c3p0-hibernate.version>6.2.2.Final</c3p0-hibernate.version> <apache-jena.version>4.8.0</apache-jena.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -92,6 +93,11 @@ <artifactId>mariadb-java-client</artifactId> <version>${mariadb.version}</version> </dependency> + <dependency> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> + </dependency> <!-- Testing --> <dependency> <groupId>org.springframework.boot</groupId> diff --git a/dbrepo-semantics-service/rest-service/src/main/resources/application-local.yml b/dbrepo-semantics-service/rest-service/src/main/resources/application-local.yml index cea3a41bb4..890c01ef05 100644 --- a/dbrepo-semantics-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-semantics-service/rest-service/src/main/resources/application-local.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic + opensearch: + username: admin + password: admin uris: http://localhost:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-semantics-service/rest-service/src/main/resources/application.yml b/dbrepo-semantics-service/rest-service/src/main/resources/application.yml index 7e35d82129..752873494f 100644 --- a/dbrepo-semantics-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-semantics-service/rest-service/src/main/resources/application.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: elastic + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" uris: http://search-db:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c..0000000000 --- a/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000..27d815cc6f --- /dev/null +++ b/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/dbrepo-table-service/pom.xml b/dbrepo-table-service/pom.xml index a044da6517..b32a4dde83 100644 --- a/dbrepo-table-service/pom.xml +++ b/dbrepo-table-service/pom.xml @@ -34,6 +34,7 @@ <c3p0.version>0.9.5.5</c3p0.version> <c3p0-hibernate.version>6.2.2.Final</c3p0-hibernate.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -120,8 +121,9 @@ <version>${c3p0-hibernate.version}</version> </dependency> <dependency> - <groupId>org.springframework.data</groupId> - <artifactId>spring-data-elasticsearch</artifactId> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> </dependency> <dependency> <groupId>org.mariadb.jdbc</groupId> diff --git a/dbrepo-table-service/rest-service/src/main/resources/application-local.yml b/dbrepo-table-service/rest-service/src/main/resources/application-local.yml index e3a4c13c91..65be9f4bda 100644 --- a/dbrepo-table-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-table-service/rest-service/src/main/resources/application-local.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic + opensearch: + username: admin + password: admin uris: http://localhost:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-table-service/rest-service/src/main/resources/application.yml b/dbrepo-table-service/rest-service/src/main/resources/application.yml index ac70fe699a..d874155e5c 100644 --- a/dbrepo-table-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-table-service/rest-service/src/main/resources/application.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: elastic + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" uris: http://search-db:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-table-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-table-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c..0000000000 --- a/dbrepo-table-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-table-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-table-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000..27d815cc6f --- /dev/null +++ b/dbrepo-table-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/dbrepo-ui/.env.example b/dbrepo-ui/.env.example index 48fd54b786..e1bec7f2bc 100644 --- a/dbrepo-ui/.env.example +++ b/dbrepo-ui/.env.example @@ -5,8 +5,8 @@ BROKER_PASSWORD="fda" DBREPO_CLIENT_SECRET="MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG" DEFAULT_PID_PUBLISHER="Technische Universität Wien" DOI_URL="https://doi.org" -ELASTIC_USERNAME="elastic" -ELASTIC_PASSWORD="elastic" +SEARCH_USERNAME="admin" +SEARCH_PASSWORD="admin" LOGO="/logo.png" SEARCH="http://localhost:3001/retrieve" SHARED_FILESYSTEM="/tmp" diff --git a/dbrepo-ui/Dockerfile b/dbrepo-ui/Dockerfile index d014add23b..4b37385c0c 100644 --- a/dbrepo-ui/Dockerfile +++ b/dbrepo-ui/Dockerfile @@ -50,8 +50,8 @@ ENV BROKER_PASSWORD="fda" ENV SEARCH="http://search-db:9200" ENV SHARED_FILESYSTEM="/tmp" ENV LOGO="/logo.png" -ENV ELASTIC_USERNAME="elastic" -ENV ELASTIC_PASSWORD="elastic" +ENV SEARCH_USERNAME="admin" +ENV SEARCH_PASSWORD="admin" ENV VERSION="${TAG}" ENV TITLE="Database Repository" ENV ICON="/favicon.ico" diff --git a/dbrepo-ui/config.js b/dbrepo-ui/config.js index b12b7b7055..e83a19c0c0 100644 --- a/dbrepo-ui/config.js +++ b/dbrepo-ui/config.js @@ -12,7 +12,7 @@ config.version = process.env.VERSION || 'latest' config.logo = process.env.LOGO || '/logo.png' config.mailVerify = process.env.MAIL_VERIFY || false config.tokenMax = process.env.TOKEN_MAX || 5 -config.elasticPassword = process.env.ELASTIC_PASSWORD || 'elastic' +config.elasticPassword = process.env.SEARCH_PASSWORD || 'elastic' config.clientId = process.env.DBREPO_CLIENT_ID || 'dbrepo-client' config.clientSecret = process.env.DBREPO_CLIENT_SECRET || 'MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG' config.defaultPublisher = process.env.DEFAULT_PID_PUBLISHER || 'Example University' diff --git a/dbrepo-user-service/Dockerfile b/dbrepo-user-service/Dockerfile index fb50823d3c..b3c9c5faa1 100644 --- a/dbrepo-user-service/Dockerfile +++ b/dbrepo-user-service/Dockerfile @@ -33,7 +33,7 @@ ENV JWT_ISSUER="http://localhost/realms/dbrepo" ENV JWT_PUBKEY="MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB" ENV LOG_LEVEL=debug ENV DEFAULT_ROLE="default-researcher-roles" -ENV ELASTIC_PASSWORD=elastic +ENV SEARCH_PASSWORD=admin WORKDIR /app diff --git a/dbrepo-user-service/pom.xml b/dbrepo-user-service/pom.xml index 30e36bcba0..dad7c33127 100644 --- a/dbrepo-user-service/pom.xml +++ b/dbrepo-user-service/pom.xml @@ -17,7 +17,7 @@ also allows for view-only (possibly paginated and versioned) query execution to the raw data and consumes messages in the message queue from the Broker Service. </description> - <url>https://dbrepo-docs.ossdip.at</url> + <url>https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/</url> <developers> <developer> <name>Martin Weise</name> @@ -52,6 +52,7 @@ <keycloak.version>21.0.2</keycloak.version> <hibernate.version>6.2.2.Final</hibernate.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -116,8 +117,9 @@ <version>${mariadb.version}</version> </dependency> <dependency> - <groupId>org.springframework.data</groupId> - <artifactId>spring-data-elasticsearch</artifactId> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> </dependency> <!-- Swagger --> <dependency> diff --git a/dbrepo-user-service/rest-service/src/main/resources/application-local.yml b/dbrepo-user-service/rest-service/src/main/resources/application-local.yml index e15f94b2cc..c0c994b5ff 100644 --- a/dbrepo-user-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-user-service/rest-service/src/main/resources/application-local.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic + opensearch: + username: admin + password: admin uris: http://localhost:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-user-service/rest-service/src/main/resources/application.yml b/dbrepo-user-service/rest-service/src/main/resources/application.yml index 58ca1c7f57..910db255f1 100644 --- a/dbrepo-user-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-user-service/rest-service/src/main/resources/application.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: elastic + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" uris: http://search-db:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-user-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-user-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c..0000000000 --- a/dbrepo-user-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-user-service/services/src/main/java/at/tuwien/config/IndexConfig.java b/dbrepo-user-service/services/src/main/java/at/tuwien/config/IndexConfig.java new file mode 100644 index 0000000000..ba3a4c634e --- /dev/null +++ b/dbrepo-user-service/services/src/main/java/at/tuwien/config/IndexConfig.java @@ -0,0 +1,53 @@ +package at.tuwien.config; + +import at.tuwien.api.user.UserDto; +import at.tuwien.mapper.UserMapper; +import at.tuwien.repository.elastic.UserIdxRepository; +import at.tuwien.repository.jpa.UserRepository; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +@Component +@Log4j2 +public class IndexConfig { + + private final UserMapper userMapper; + private final UserRepository userRepository; + private final UserIdxRepository userIdxRepository; + private final ElasticsearchOperations elasticsearchOperations; + + @Autowired + public IndexConfig(UserMapper userMapper, UserRepository userRepository, + UserIdxRepository userIdxRepository, ElasticsearchOperations elasticsearchOperations) { + this.userMapper = userMapper; + this.userRepository = userRepository; + this.userIdxRepository = userIdxRepository; + this.elasticsearchOperations = elasticsearchOperations; + } + + @Transactional + @EventListener(ApplicationReadyEvent.class) + public void initIndex() { + final IndexCoordinates userIndex = IndexCoordinates.of("user"); + if (!elasticsearchOperations.indexOps(userIndex).exists()) { + elasticsearchOperations.indexOps(userIndex).create(); + elasticsearchOperations.indexOps(userIndex).createMapping(UserDto.class); + log.info("Created user index"); + } + /* pre-fill */ + final List<UserDto> users = userRepository.findAll() + .stream() + .map(userMapper::userToUserDto) + .toList(); + userIdxRepository.saveAll(users); + log.info("Added {} users to OpenSearch index", users.size()); + } +} diff --git a/dbrepo-user-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-user-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000..27d815cc6f --- /dev/null +++ b/dbrepo-user-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 909b52556b..facea5fad7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -296,7 +296,7 @@ services: restart: always container_name: dbrepo-search-db hostname: search-db - image: elasticsearch:8.7.1 + image: opensearchproject/opensearch:2 networks: core: ports: @@ -305,10 +305,14 @@ services: - .env environment: discovery.type: "single-node" - ES_JAVA_OPTS: "-Xms2g -Xmx2g" + ES_JAVA_OPTS: "-Xms4g -Xmx4g" logger.level: "WARN" + plugins.security.disabled: "true" bootstrap.memory_lock: "true" - xpack.security.enabled: "true" + deploy: + resources: + limits: + memory: 4G volumes: - search-db-data:/usr/share/elasticsearch/data logging: -- GitLab