diff --git a/.env.unix.example b/.env.unix.example index 5e3cb074dc4c043b4a17bfd606ceb852a1f658d5..e527bca539e3ba4ea35a87269ce4084e6e38c300 100644 --- a/.env.unix.example +++ b/.env.unix.example @@ -4,7 +4,8 @@ JWT_ISSUER="http://localhost/realms/dbrepo" JWT_PUBKEY="MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB" JWT_CERT="MIICmzCCAYMCBgGG3GWyBTANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZkYnJlcG8wHhcNMjMwMzEzMTkxMzE3WhcNMzMwMzEzMTkxNDU3WjARMQ8wDQYDVQQDDAZkYnJlcG8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCqqcdDYFZZb28M0tEJzEP77FmD/Xqioyj9zWX6VwUSOMAgmMmn8eqs9hT9T0a+q4YTo9tUW1PNbUpwprA5b4Uk04DcIajxDVMUR/PjcHytmkqwVskq9AZW/Vngdoo+8tSbuIybwe/3Vwt266hbHpDcM97a+DXcYooRl7tQWCEX7RP27wQrMD9epDQ6IgKayZg9vC9/03dsIqwH9jXQRiZlFvwiEKhX2aY7lPGBaCK414JO00K/Z49iov9TRa/IYVbSt5qwgrx6DcqsBSPwOnI6A85UGfeUEZ/7coVJiL7RvBlsllapsL9eWTbQajVh94k9Ei3sibEPbtH+U2OAM78zAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAASnN1Cuif1sdfEK2kWAURSXGJCohCROLWdKFjaeHPRaEfpbFJsgxW0Yj3nwX5O3bUlOWoTyENwnXSsXMQsqnNi+At32CKaKO8+AkhAbgQL9F0B+KeJwmYv3cUj5N/LYkJjBvZBzUZ4Ugu5dcxH0k7AktLAIwimkyEnxTNolOA3UyrGGpREr8MCKWVr10RFuOpF/0CsJNNwbHXzalO9D756EUcRWZ9VSg6QVNso0YYRKTnILWDn9hcTRnqGy3SHo3anFTqQZ+BB57YbgFWy6udC0LYRB3zdp6zNti87eu/VEymiDY/mmo1AB8Tm0b6vxFz4AKcL3ax5qS6YnZ9efSzk=" SHARED_FILESYSTEM=/tmp -ELASTIC_PASSWORD=elastic +SEARCH_USERNAME=admin +SEARCH_PASSWORD=admin METADATA_DB=fda METADATA_USERNAME=root METADATA_PASSWORD=dbrepo diff --git a/README.md b/README.md index 1fde55325b7c626ff08179ba7411a8e01816aeee..337f5d812907c18e02e495b0b2e7ce5d42e07d94 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ $ docker-compose build --parallel A more detailed description on how to get started is available at our documentation -website: [https://dbrepo-docs.ossdip.at/getting-started/](https://dbrepo-docs.ossdip.at/getting-started/) +website: [https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//getting-started/](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//getting-started/) ### Run diff --git a/dbrepo-container-service/Dockerfile b/dbrepo-container-service/Dockerfile index 9e8c839383c3441553d11fe8aeb473133f7e3b82..9b27413bd73028ad07a0ca6ec182afa6f2131d7c 100644 --- a/dbrepo-container-service/Dockerfile +++ b/dbrepo-container-service/Dockerfile @@ -30,8 +30,8 @@ ENV METADATA_USERNAME=root ENV METADATA_PASSWORD=dbrepo ENV BROKER_USERNAME=fda ENV BROKER_PASSWORD=fda -ENV ELASTIC_USERNAME=elastic -ENV ELASTIC_PASSWORD=elastic +ENV SEARCH_USERNAME=admin +ENV SEARCH_PASSWORD=admin ENV SHARED_FILESYSTEM=/tmp ENV USER_NETWORK=userdb ENV LOG_LEVEL=debug diff --git a/dbrepo-container-service/pom.xml b/dbrepo-container-service/pom.xml index c17167494db72c163d1304779485e5cf338e92f0..8c4c65ef615f46ca8d19fcf3d980e91f23016a65 100644 --- a/dbrepo-container-service/pom.xml +++ b/dbrepo-container-service/pom.xml @@ -30,6 +30,7 @@ <jacoco.version>0.8.10</jacoco.version> <jwt.version>4.3.0</jwt.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch-client.version>1.1.0</opensearch-client.version> </properties> <dependencies> @@ -88,6 +89,16 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-jpa</artifactId> </dependency> + <dependency> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch</artifactId> + <version>${opensearch-client.version}</version> + </dependency> + <dependency> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch-client.version}</version> + </dependency> <!-- Entity and API --> <dependency> <groupId>at.tuwien</groupId> @@ -108,6 +119,12 @@ <version>${mariadb.version}</version> </dependency> <!-- Testing --> + <dependency> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-test-autoconfigure</artifactId> + <version>${opensearch-client.version}</version> + <scope>test</scope> + </dependency> <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> diff --git a/dbrepo-container-service/rest-service/src/main/resources/application-local.yml b/dbrepo-container-service/rest-service/src/main/resources/application-local.yml index 9760a7ba9def6da76170e5f68bdb68903ec12d13..2af1c6bc21f1e8da7a8076cf4def0a054fe07887 100644 --- a/dbrepo-container-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-container-service/rest-service/src/main/resources/application-local.yml @@ -27,10 +27,6 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic - uris: http://localhost:9200 management.endpoints.web.exposure.include: health,info,prometheus springdoc.swagger-ui.enabled: true server: @@ -50,4 +46,8 @@ fda: public_key: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB client_secret: client-secret client_id: dbrepo-client - gateway.endpoint: http://localhost \ No newline at end of file + gateway.endpoint: http://localhost + opensearch: + username: admin + password: admin + uris: http://localhost:9200 \ No newline at end of file diff --git a/dbrepo-container-service/rest-service/src/main/resources/application.yml b/dbrepo-container-service/rest-service/src/main/resources/application.yml index 70619e94b9d9ea34578c06eb8e1fc42b38b7ebb0..907baa318fed02709f6ec6ad59a06e9e526b4ed4 100644 --- a/dbrepo-container-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-container-service/rest-service/src/main/resources/application.yml @@ -27,10 +27,6 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: "${ELASTIC_USERNAME}" - uris: http://search-db:9200 management.endpoints.web.exposure.include: health,info,prometheus springdoc.swagger-ui.enabled: true server: @@ -50,4 +46,8 @@ fda: public_key: "${JWT_PUBKEY}" client_secret: "${DBREPO_CLIENT_SECRET}" client_id: "${CLIENT_ID}" - gateway.endpoint: http://gateway-service \ No newline at end of file + gateway.endpoint: http://gateway-service + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" + uris: http://search-db:9200 \ No newline at end of file diff --git a/dbrepo-database-service/Dockerfile b/dbrepo-database-service/Dockerfile index bf79827ee7dd87c0b008b564516ac582764a3c5e..c634dc395645309c3cf74124080c5449848a1e7c 100644 --- a/dbrepo-database-service/Dockerfile +++ b/dbrepo-database-service/Dockerfile @@ -30,8 +30,8 @@ ENV METADATA_USERNAME=root ENV METADATA_PASSWORD=dbrepo ENV BROKER_USERNAME=fda ENV BROKER_PASSWORD=fda -ENV ELASTIC_USERNAME=elastic -ENV ELASTIC_PASSWORD=elastic +ENV SEARCH_USERNAME=admin +ENV SEARCH_PASSWORD=admin ENV GATEWAY_ENDPOINT=http://gateway-service ENV LOG_LEVEL=debug ENV CLIENT_ID="dbrepo-client" diff --git a/dbrepo-database-service/pom.xml b/dbrepo-database-service/pom.xml index d694d035a95e4122a827f9a099729427a7acffd5..021bcf9c1ab52e30f983c5a3c5e21822a5740cc6 100644 --- a/dbrepo-database-service/pom.xml +++ b/dbrepo-database-service/pom.xml @@ -31,6 +31,7 @@ <c3p0.version>0.9.5.5</c3p0.version> <c3p0-hibernate.version>6.2.2.Final</c3p0-hibernate.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -69,9 +70,9 @@ </dependency> <!-- elasticsearch --> <dependency> - <groupId>co.elastic.clients</groupId> - <artifactId>elasticsearch-java</artifactId> - <version>${elasticsearch-client.version}</version> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> </dependency> <!-- AMQP --> <dependency> diff --git a/dbrepo-database-service/rest-service/src/main/resources/application.yml b/dbrepo-database-service/rest-service/src/main/resources/application.yml index ba7ad2434e03693d0413759fcec3c985c3984586..3202f76b38a602595e86393a7515c975466ed904 100644 --- a/dbrepo-database-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-database-service/rest-service/src/main/resources/application.yml @@ -27,9 +27,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: "${ELASTIC_USERNAME}" + opensearch: + password: "${SEARCH_PASSWORD}" + username: "${SEARCH_USERNAME}" uris: http://search-db:9200 management.endpoints.web.exposure.include: health,info,prometheus server: diff --git a/dbrepo-database-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-database-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c7c08d8c0ae75ee1ecf0191a7d19f65..0000000000000000000000000000000000000000 --- a/dbrepo-database-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-database-service/services/src/main/java/at/tuwien/config/IndexConfig.java b/dbrepo-database-service/services/src/main/java/at/tuwien/config/IndexConfig.java index dc0e46355d1aa2cc10ed1c2137bba891a95ad4e2..69e950bac1a08554474e66555fdcdb5d779946ab 100644 --- a/dbrepo-database-service/services/src/main/java/at/tuwien/config/IndexConfig.java +++ b/dbrepo-database-service/services/src/main/java/at/tuwien/config/IndexConfig.java @@ -21,38 +21,33 @@ import java.util.stream.Collectors; @Component public class IndexConfig { - private final Environment environment; private final DatabaseMapper databaseMapper; private final DatabaseRepository databaseRepository; private final DatabaseIdxRepository databaseIdxRepository; private final ElasticsearchOperations elasticsearchOperations; - private final ApplicationEventPublisher applicationEventPublisher; - public IndexConfig(Environment environment, DatabaseMapper databaseMapper, DatabaseRepository databaseRepository, - DatabaseIdxRepository databaseIdxRepository, ElasticsearchOperations elasticsearchOperations, - ApplicationEventPublisher applicationEventPublisher) { - this.environment = environment; + public IndexConfig(DatabaseMapper databaseMapper, DatabaseRepository databaseRepository, + DatabaseIdxRepository databaseIdxRepository, ElasticsearchOperations elasticsearchOperations) { this.databaseMapper = databaseMapper; this.databaseRepository = databaseRepository; this.databaseIdxRepository = databaseIdxRepository; this.elasticsearchOperations = elasticsearchOperations; - this.applicationEventPublisher = applicationEventPublisher; } @Transactional @EventListener(ApplicationReadyEvent.class) public void initIndex() { - log.debug("creating databaseindex"); - final IndexCoordinates databaseIndex = IndexCoordinates.of("databaseindex"); + final IndexCoordinates databaseIndex = IndexCoordinates.of("database"); if (!elasticsearchOperations.indexOps(databaseIndex).exists()) { elasticsearchOperations.indexOps(databaseIndex).create(); elasticsearchOperations.indexOps(databaseIndex).createMapping(DatabaseDto.class); + log.info("Created identifier index"); } final List<DatabaseDto> databases = databaseRepository.findAll() .stream() .map(databaseMapper::databaseToDatabaseDto) .collect(Collectors.toList()); - log.debug("add {} databases to elastic search index", databases.size()); + log.info("Added {} databases to OpenSearch index", databases.size()); databaseIdxRepository.saveAll(databases); } diff --git a/dbrepo-database-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-database-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..27d815cc6f4329c5374c31fb5299ccabedf24d39 --- /dev/null +++ b/dbrepo-database-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/dbrepo-identifier-service/pom.xml b/dbrepo-identifier-service/pom.xml index 07fe813de20df93f19cc15894d164db2c717507d..ef0bed4d9d6b0b807951db97458bdadf29c0d6e9 100644 --- a/dbrepo-identifier-service/pom.xml +++ b/dbrepo-identifier-service/pom.xml @@ -35,6 +35,7 @@ <super-csv.version>2.4.0</super-csv.version> <jsql.version>4.6</jsql.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -98,8 +99,9 @@ <version>${mariadb.version}</version> </dependency> <dependency> - <groupId>org.springframework.data</groupId> - <artifactId>spring-data-elasticsearch</artifactId> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> </dependency> <!-- Docker --> <dependency> diff --git a/dbrepo-identifier-service/rest-service/src/main/resources/application-local.yml b/dbrepo-identifier-service/rest-service/src/main/resources/application-local.yml index 6800891595f1666ccdf046f764174ac52ccf7ca6..15a36b4bea2db5c56fcbe8330d1e267cc2bfd678 100644 --- a/dbrepo-identifier-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-identifier-service/rest-service/src/main/resources/application-local.yml @@ -27,9 +27,9 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic + opensearch: + username: admin + password: admin uris: http://localhost:9200 management.endpoints.web.exposure.include: health,info,prometheus server: diff --git a/dbrepo-identifier-service/rest-service/src/main/resources/application.yml b/dbrepo-identifier-service/rest-service/src/main/resources/application.yml index 0ec8f69c7b680fb1105deb024141aa139aff4333..23fcfe27270363672c9a0c892c414fbcf7e3fa95 100644 --- a/dbrepo-identifier-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-identifier-service/rest-service/src/main/resources/application.yml @@ -27,9 +27,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: elastic + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" uris: http://search-db:9200 management.endpoints.web.exposure.include: health,info,prometheus server: diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/ActuatorComponentTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/ActuatorComponentTest.java index 3677c7f0451b17bc696a779ff759066bfd57832d..2f817c1ec836e3db275c721989c45dafd5cd70af 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/ActuatorComponentTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/ActuatorComponentTest.java @@ -1,7 +1,7 @@ package at.tuwien.endpoint; import at.tuwien.BaseUnitTest; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import lombok.extern.log4j.Log4j2; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -26,7 +26,7 @@ public class ActuatorComponentTest extends BaseUnitTest { private MockMvc mockMvc; @MockBean - private IndexInitializer indexConfig; + private IndexConfig indexConfig; @Test public void actuatorInfo_succeeds() throws Exception { diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointIntegrationTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointIntegrationTest.java index a2ea25e54284cb6df30fa4558a3724832bf7fc9e..ddd2f989f2380a477f1f3a5e75e7cf5530a06f33 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointIntegrationTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointIntegrationTest.java @@ -3,7 +3,7 @@ package at.tuwien.endpoint; import at.tuwien.BaseUnitTest; import at.tuwien.api.identifier.IdentifierDto; import at.tuwien.api.identifier.IdentifierTypeDto; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.config.ReadyConfig; import at.tuwien.endpoints.IdentifierEndpoint; import at.tuwien.exception.NotAllowedException; @@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.*; public class IdentifierEndpointIntegrationTest extends BaseUnitTest { @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private ReadyConfig readyConfig; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointUnitTest.java index 1be9690cb0e989e974a8f81494a98ccd68acf202..f94307ae80a9adbf22692d0bc55c30f62958b365 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointUnitTest.java @@ -5,7 +5,7 @@ import at.tuwien.api.identifier.IdentifierCreateDto; import at.tuwien.api.identifier.IdentifierDto; import at.tuwien.api.identifier.IdentifierTypeDto; import at.tuwien.config.EndpointConfig; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.config.ReadyConfig; import at.tuwien.endpoints.IdentifierEndpoint; import at.tuwien.endpoints.PersistenceEndpoint; @@ -29,7 +29,6 @@ import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.AccessDeniedException; -import org.springframework.security.authentication.AuthenticationCredentialsNotFoundException; import org.springframework.security.test.context.support.WithAnonymousUser; import org.springframework.security.test.context.support.WithMockUser; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -54,7 +53,7 @@ public class IdentifierEndpointUnitTest extends BaseUnitTest { private ReadyConfig readyConfig; @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private IdentifierService identifierService; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/PersistenceEndpointUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/PersistenceEndpointUnitTest.java index d3c64bd929d72368c2917ed25f952fce282391b7..7e9c11f73dd68621069b8a396a1d9627d7762e52 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/PersistenceEndpointUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/PersistenceEndpointUnitTest.java @@ -5,7 +5,7 @@ import at.tuwien.api.identifier.BibliographyTypeDto; import at.tuwien.api.identifier.CreatorDto; import at.tuwien.api.identifier.IdentifierDto; import at.tuwien.api.identifier.IdentifierUpdateDto; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.config.ReadyConfig; import at.tuwien.endpoints.PersistenceEndpoint; import at.tuwien.entities.identifier.Identifier; @@ -54,7 +54,7 @@ public class PersistenceEndpointUnitTest extends BaseUnitTest { private ReadyConfig readyConfig; @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private AccessService accessService; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/SwaggerComponentTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/SwaggerComponentTest.java index 0caa396455da3088708d66147dfe1afd35ae981a..47a13598411493feb0bee24548014427ecb856f9 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/SwaggerComponentTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/SwaggerComponentTest.java @@ -1,7 +1,7 @@ package at.tuwien.endpoint; import at.tuwien.BaseUnitTest; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import lombok.extern.log4j.Log4j2; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -26,7 +26,7 @@ public class SwaggerComponentTest extends BaseUnitTest { private MockMvc mockMvc; @MockBean - private IndexInitializer indexConfig; + private IndexConfig indexConfig; @Test public void swaggerUi_succeeds() throws Exception { diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/gateway/QueryServiceGatewayUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/gateway/QueryServiceGatewayUnitTest.java index 1a68af0041864473bcb23c43e0679a2c90d21f2d..6a39ebb85b9bfcc9479e4eac034f6f5c20278c1e 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/gateway/QueryServiceGatewayUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/gateway/QueryServiceGatewayUnitTest.java @@ -2,7 +2,7 @@ package at.tuwien.gateway; import at.tuwien.BaseUnitTest; import at.tuwien.api.database.query.QueryDto; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.config.ReadyConfig; import at.tuwien.exception.QueryNotFoundException; import at.tuwien.exception.RemoteUnavailableException; @@ -36,7 +36,7 @@ import static org.mockito.Mockito.*; public class QueryServiceGatewayUnitTest extends BaseUnitTest { @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private ReadyConfig readyConfig; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/DataCiteIdentifierServiceUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/DataCiteIdentifierServiceUnitTest.java index 1aad8f0b61650640eaeb10bb52234b2c19d6411b..4bc2800a8f08f96a257188951617469904c36982 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/DataCiteIdentifierServiceUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/DataCiteIdentifierServiceUnitTest.java @@ -8,7 +8,7 @@ import at.tuwien.api.identifier.IdentifierCreateDto; import at.tuwien.api.identifier.IdentifierUpdateDto; import at.tuwien.config.DataCiteConfig; import at.tuwien.config.EndpointConfig; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.entities.identifier.Identifier; import at.tuwien.exception.*; import at.tuwien.repository.jpa.*; @@ -29,7 +29,6 @@ import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; @@ -47,7 +46,7 @@ import static org.mockito.Mockito.when; public class DataCiteIdentifierServiceUnitTest extends BaseUnitTest { @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean(answer = Answers.RETURNS_MOCKS) private DataCiteConfig dataCiteConfig; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceIntegrationTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceIntegrationTest.java index 27bdb8c69b75578f56b203ec6f5fc9b0a570b136..8b826cb105bf8d554f21fcd9ae61044d7156edf3 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceIntegrationTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceIntegrationTest.java @@ -3,7 +3,7 @@ package at.tuwien.service; import at.tuwien.BaseUnitTest; import at.tuwien.api.database.query.QueryDto; import at.tuwien.api.identifier.IdentifierDto; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.entities.identifier.Identifier; import at.tuwien.entities.identifier.RelatedIdentifier; import at.tuwien.exception.*; @@ -39,7 +39,7 @@ import static org.mockito.Mockito.when; public class IdentifierServiceIntegrationTest extends BaseUnitTest { @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private IdentifierIdxRepository identifierIdxRepository; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceUnitTest.java index d7e8b28963278b32426509a2bffbe4816018a839..b44d0282f571bfbb087d38d91ee8d55d3ce4d2fc 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/IdentifierServiceUnitTest.java @@ -3,8 +3,7 @@ package at.tuwien.service; import at.tuwien.BaseUnitTest; import at.tuwien.api.database.query.QueryDto; import at.tuwien.api.identifier.IdentifierDto; -import at.tuwien.api.identifier.IdentifierUpdateDto; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.entities.identifier.Identifier; import at.tuwien.entities.identifier.IdentifierType; import at.tuwien.exception.*; @@ -39,7 +38,7 @@ public class IdentifierServiceUnitTest extends BaseUnitTest { private IdentifierRepository identifierRepository; @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private IdentifierIdxRepository identifierIdxRepository; diff --git a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java index da772b10e1d1e3ba7830eacddf2ef2e352031b55..baab96d40cc35d4136576162d84fcadbf6d61d5f 100644 --- a/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java +++ b/dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java @@ -1,7 +1,7 @@ package at.tuwien.service; import at.tuwien.BaseUnitTest; -import at.tuwien.config.IndexInitializer; +import at.tuwien.config.IndexConfig; import at.tuwien.entities.user.User; import at.tuwien.exception.*; import at.tuwien.repository.jpa.UserRepository; @@ -23,7 +23,7 @@ import static org.mockito.Mockito.when; public class UserServiceUnitTest extends BaseUnitTest { @MockBean - private IndexInitializer indexInitializer; + private IndexConfig indexInitializer; @MockBean private UserRepository userRepository; diff --git a/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c7c08d8c0ae75ee1ecf0191a7d19f65..0000000000000000000000000000000000000000 --- a/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexInitializer.java b/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexConfig.java similarity index 81% rename from dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexInitializer.java rename to dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexConfig.java index 0592364d2de1925f1f1298e635c3ddc58d0e0e06..2f0a40e3b410ff2469e7df01b785d5e60c86ff68 100644 --- a/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexInitializer.java +++ b/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/IndexConfig.java @@ -18,7 +18,7 @@ import java.util.stream.Collectors; @Component @Log4j2 -public class IndexInitializer { +public class IndexConfig { private final IdentifierMapper identifierMapper; private final IdentifierRepository identifierRepository; @@ -26,9 +26,9 @@ public class IndexInitializer { private final ElasticsearchOperations elasticsearchOperations; @Autowired - public IndexInitializer(IdentifierMapper identifierMapper, IdentifierRepository identifierRepository, - IdentifierIdxRepository identifierIdxRepository, - ElasticsearchOperations elasticsearchOperations) { + public IndexConfig(IdentifierMapper identifierMapper, IdentifierRepository identifierRepository, + IdentifierIdxRepository identifierIdxRepository, + ElasticsearchOperations elasticsearchOperations) { this.identifierMapper = identifierMapper; this.identifierRepository = identifierRepository; this.identifierIdxRepository = identifierIdxRepository; @@ -38,17 +38,17 @@ public class IndexInitializer { @Transactional @EventListener(ApplicationReadyEvent.class) public void initIndex() { - log.debug("creating identifierindex"); - final IndexCoordinates identifierIndex = IndexCoordinates.of("identifierindex"); + final IndexCoordinates identifierIndex = IndexCoordinates.of("identifier"); if (!elasticsearchOperations.indexOps(identifierIndex).exists()) { elasticsearchOperations.indexOps(identifierIndex).create(); elasticsearchOperations.indexOps(identifierIndex).createMapping(IdentifierDto.class); + log.info("Created identifier index"); } final List<IdentifierDto> identifiers = identifierRepository.findAll() .stream() .map(identifierMapper::identifierToIdentifierDto) .collect(Collectors.toList()); - log.debug("add {} identifiers to elastic search index", identifiers.size()); + log.info("Add {} identifiers to OpenSearch index", identifiers.size()); identifierIdxRepository.saveAll(identifiers); } } diff --git a/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..27d815cc6f4329c5374c31fb5299ccabedf24d39 --- /dev/null +++ b/dbrepo-identifier-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/dbrepo-metadata-db/pom.xml b/dbrepo-metadata-db/pom.xml index cccf8f2178cd6466e0b04f757816004303a4c40a..284747e3d7382d842dc6b96c51ae96dd0c91b72f 100644 --- a/dbrepo-metadata-db/pom.xml +++ b/dbrepo-metadata-db/pom.xml @@ -32,6 +32,7 @@ <jackson-datatype.version>2.15.0</jackson-datatype.version> <commons.version>2.11.0</commons.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -52,6 +53,11 @@ <groupId>org.springframework.data</groupId> <artifactId>spring-data-elasticsearch</artifactId> </dependency> + <dependency> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> + </dependency> <!-- Mapping --> <dependency> <groupId>com.fasterxml.jackson.datatype</groupId> diff --git a/dbrepo-metadata-db/test/src/main/java/at/tuwien/test/BaseTest.java b/dbrepo-metadata-db/test/src/main/java/at/tuwien/test/BaseTest.java index 22e0c0c39ac2e5830512adf7099c58c6307a2e1d..c9874c5799e168d853f930b0fafce9a497805f51 100644 --- a/dbrepo-metadata-db/test/src/main/java/at/tuwien/test/BaseTest.java +++ b/dbrepo-metadata-db/test/src/main/java/at/tuwien/test/BaseTest.java @@ -1118,7 +1118,7 @@ public abstract class BaseTest { public final static String CONTAINER_ELASTIC_HASH = "deadbeef"; public final static Instant CONTAINER_ELASTIC_CREATED = Instant.ofEpochSecond(1677399721) /* 2023-02-26 08:22:01 (UTC) */; public final static String[] CONTAINER_ELASTIC_ENV = new String[]{"discovery.type=single-node", "ES_JAVA_OPTS=-Xms2g -Xmx2g", - "logger.level=WARN", "bootstrap.memory_lock=true", "xpack.security.enabled=true", "ELASTIC_PASSWORD=elastic"}; + "logger.level=WARN", "bootstrap.memory_lock=true", "xpack.security.enabled=true", "SEARCH_PASSWORD=admin"}; public final static Container CONTAINER_ELASTIC = Container.builder() .id(CONTAINER_ELASTIC_ID) diff --git a/dbrepo-metadata-service/pom.xml b/dbrepo-metadata-service/pom.xml index 261784bc3f666e7ab4a831ce26edbd99a6baf06b..beddd0a5dab1364701d38515faa623232e662723 100644 --- a/dbrepo-metadata-service/pom.xml +++ b/dbrepo-metadata-service/pom.xml @@ -14,7 +14,7 @@ <name>dbrepo-metadata-service</name> <description>Service that manages the metadata</description> - <url>https://dbrepo-docs.ossdip.at</url> + <url>https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/</url> <developers> <developer> <name>Martin Weise</name> diff --git a/dbrepo-query-service/pom.xml b/dbrepo-query-service/pom.xml index 095adfeb17d64c98ef2f078fa41c0c3973cf5e19..9274612bde7dd3c35118aa5b24cba860ebc72da4 100644 --- a/dbrepo-query-service/pom.xml +++ b/dbrepo-query-service/pom.xml @@ -17,7 +17,7 @@ also allows for view-only (possibly paginated and versioned) query execution to the raw data and consumes messages in the message queue from the Broker Service. </description> - <url>https://dbrepo-docs.ossdip.at</url> + <url>https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/</url> <developers> <developer> <name>Martin Weise</name> @@ -51,6 +51,7 @@ <c3p0.version>0.9.5.5</c3p0.version> <c3p0-hibernate.version>6.2.2.Final</c3p0-hibernate.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -121,8 +122,9 @@ <version>${mariadb.version}</version> </dependency> <dependency> - <groupId>org.springframework.data</groupId> - <artifactId>spring-data-elasticsearch</artifactId> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> </dependency> <!-- AMPQ --> <dependency> diff --git a/dbrepo-query-service/rest-service/src/main/resources/application-local.yml b/dbrepo-query-service/rest-service/src/main/resources/application-local.yml index 32b8a33c40da0168e779a5b2ae3622107e623214..ee811a1e255c7b3082cab7e96567170f6ddb17bd 100644 --- a/dbrepo-query-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-query-service/rest-service/src/main/resources/application-local.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic + opensearch: + username: admin + password: admin uris: http://localhost:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-query-service/rest-service/src/main/resources/application.yml b/dbrepo-query-service/rest-service/src/main/resources/application.yml index db37d874ec942755cc879f26242862f6330ca488..5caaab1e9b62a38c7f9d08fdac089e563143a835 100644 --- a/dbrepo-query-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-query-service/rest-service/src/main/resources/application.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: elastic + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" uris: http://search-db:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-query-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-query-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c7c08d8c0ae75ee1ecf0191a7d19f65..0000000000000000000000000000000000000000 --- a/dbrepo-query-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-query-service/services/src/main/java/at/tuwien/config/IndexConfig.java b/dbrepo-query-service/services/src/main/java/at/tuwien/config/IndexConfig.java index eeb61eac82301fc0030619cfa3a3c6d53759109d..5b9b65d3dd045143cdcd3c8d2ffcbd7d2e806730 100644 --- a/dbrepo-query-service/services/src/main/java/at/tuwien/config/IndexConfig.java +++ b/dbrepo-query-service/services/src/main/java/at/tuwien/config/IndexConfig.java @@ -8,8 +8,6 @@ import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; -import org.springframework.core.env.Environment; -import org.springframework.core.env.Profiles; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; import org.springframework.stereotype.Component; @@ -23,16 +21,14 @@ import java.util.stream.Collectors; public class IndexConfig { private final ViewMapper viewMapper; - private final Environment environment; private final ViewRepository viewRepository; private final ViewIdxRepository viewIdxRepository; private final ElasticsearchOperations elasticsearchOperations; @Autowired - public IndexConfig(ViewMapper viewMapper, Environment environment, ViewRepository viewRepository, - ViewIdxRepository viewIdxRepository, ElasticsearchOperations elasticsearchOperations) { + public IndexConfig(ViewMapper viewMapper, ViewRepository viewRepository, ViewIdxRepository viewIdxRepository, + ElasticsearchOperations elasticsearchOperations) { this.viewMapper = viewMapper; - this.environment = environment; this.viewRepository = viewRepository; this.viewIdxRepository = viewIdxRepository; this.elasticsearchOperations = elasticsearchOperations; @@ -41,21 +37,18 @@ public class IndexConfig { @Transactional @EventListener(ApplicationReadyEvent.class) public void initIndex() { - if (environment.acceptsProfiles(Profiles.of("test-noelastic"))) { - return; - } - log.debug("creating viewindex"); - final IndexCoordinates viewIndex = IndexCoordinates.of("viewindex"); + final IndexCoordinates viewIndex = IndexCoordinates.of("view"); if (!elasticsearchOperations.indexOps(viewIndex).exists()) { elasticsearchOperations.indexOps(viewIndex).create(); elasticsearchOperations.indexOps(viewIndex).createMapping(ViewDto.class); + log.info("Created identifier index"); } /* pre-fill */ final List<ViewDto> views = viewRepository.findAll() .stream() .map(viewMapper::viewToViewDto) .collect(Collectors.toList()); - log.debug("add {} views to elastic search index", views.size()); + log.info("Added {} views to OpenSearch index", views.size()); viewIdxRepository.saveAll(views); } } diff --git a/dbrepo-query-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-query-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..27d815cc6f4329c5374c31fb5299ccabedf24d39 --- /dev/null +++ b/dbrepo-query-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/dbrepo-search-db/README.md b/dbrepo-search-db/README.md new file mode 100644 index 0000000000000000000000000000000000000000..541ebd85974b41a2072a20127cd55d6de2f2a491 --- /dev/null +++ b/dbrepo-search-db/README.md @@ -0,0 +1,10 @@ + +# Search Database + +## How to run + +Increase memory for Docker to at least 4GB: + +```console +sudo sysctl -w vm.max_map_count=262144 +``` \ No newline at end of file diff --git a/dbrepo-semantics-service/pom.xml b/dbrepo-semantics-service/pom.xml index 3eed3ff468a3e1d41e2444bc07b74db768f4c8d1..08ba51d910c8f311f4cfc1329c7748e1ef302f55 100644 --- a/dbrepo-semantics-service/pom.xml +++ b/dbrepo-semantics-service/pom.xml @@ -36,6 +36,7 @@ <c3p0-hibernate.version>6.2.2.Final</c3p0-hibernate.version> <apache-jena.version>4.8.0</apache-jena.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -92,6 +93,11 @@ <artifactId>mariadb-java-client</artifactId> <version>${mariadb.version}</version> </dependency> + <dependency> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> + </dependency> <!-- Testing --> <dependency> <groupId>org.springframework.boot</groupId> diff --git a/dbrepo-semantics-service/rest-service/src/main/resources/application-local.yml b/dbrepo-semantics-service/rest-service/src/main/resources/application-local.yml index cea3a41bb4f0e062040fe0eaa35a95246d66a0ae..890c01ef053bd079ba9e72aa61d0563ce05d8700 100644 --- a/dbrepo-semantics-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-semantics-service/rest-service/src/main/resources/application-local.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic + opensearch: + username: admin + password: admin uris: http://localhost:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-semantics-service/rest-service/src/main/resources/application.yml b/dbrepo-semantics-service/rest-service/src/main/resources/application.yml index 7e35d82129116a29148421bf5d86238d1d6e97cd..752873494fb5c13501be4bfba8307dcec880099c 100644 --- a/dbrepo-semantics-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-semantics-service/rest-service/src/main/resources/application.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: elastic + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" uris: http://search-db:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c7c08d8c0ae75ee1ecf0191a7d19f65..0000000000000000000000000000000000000000 --- a/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..27d815cc6f4329c5374c31fb5299ccabedf24d39 --- /dev/null +++ b/dbrepo-semantics-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/dbrepo-table-service/pom.xml b/dbrepo-table-service/pom.xml index a044da6517ab1c9faa02e61d019ab66831c47709..b32a4dde835ecc121bc6ea639b0a44e6e3231841 100644 --- a/dbrepo-table-service/pom.xml +++ b/dbrepo-table-service/pom.xml @@ -34,6 +34,7 @@ <c3p0.version>0.9.5.5</c3p0.version> <c3p0-hibernate.version>6.2.2.Final</c3p0-hibernate.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -120,8 +121,9 @@ <version>${c3p0-hibernate.version}</version> </dependency> <dependency> - <groupId>org.springframework.data</groupId> - <artifactId>spring-data-elasticsearch</artifactId> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> </dependency> <dependency> <groupId>org.mariadb.jdbc</groupId> diff --git a/dbrepo-table-service/rest-service/src/main/resources/application-local.yml b/dbrepo-table-service/rest-service/src/main/resources/application-local.yml index e3a4c13c91d34dbc066042c4c9a16b965bf53d66..65be9f4bdabcd1319d16e2c76ccdf1e59e8cb488 100644 --- a/dbrepo-table-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-table-service/rest-service/src/main/resources/application-local.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic + opensearch: + username: admin + password: admin uris: http://localhost:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-table-service/rest-service/src/main/resources/application.yml b/dbrepo-table-service/rest-service/src/main/resources/application.yml index ac70fe699af16b1c71725a7a8afe4c5737c46048..d874155e5c2f42be6f01479f49fad37c09ccc5e1 100644 --- a/dbrepo-table-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-table-service/rest-service/src/main/resources/application.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: elastic + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" uris: http://search-db:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-table-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-table-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c7c08d8c0ae75ee1ecf0191a7d19f65..0000000000000000000000000000000000000000 --- a/dbrepo-table-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-table-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-table-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..27d815cc6f4329c5374c31fb5299ccabedf24d39 --- /dev/null +++ b/dbrepo-table-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/dbrepo-ui/.env.example b/dbrepo-ui/.env.example index 48fd54b7861d0700d7203c50231c6138cdb39657..e1bec7f2bca376be3f8bb9ed34105851f0e3acb0 100644 --- a/dbrepo-ui/.env.example +++ b/dbrepo-ui/.env.example @@ -5,8 +5,8 @@ BROKER_PASSWORD="fda" DBREPO_CLIENT_SECRET="MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG" DEFAULT_PID_PUBLISHER="Technische Universität Wien" DOI_URL="https://doi.org" -ELASTIC_USERNAME="elastic" -ELASTIC_PASSWORD="elastic" +SEARCH_USERNAME="admin" +SEARCH_PASSWORD="admin" LOGO="/logo.png" SEARCH="http://localhost:3001/retrieve" SHARED_FILESYSTEM="/tmp" diff --git a/dbrepo-ui/Dockerfile b/dbrepo-ui/Dockerfile index d014add23bfd9be2dbf766c8cfc0d0b9d72e958d..4b37385c0c034301c7e48047608f5adb908083bf 100644 --- a/dbrepo-ui/Dockerfile +++ b/dbrepo-ui/Dockerfile @@ -50,8 +50,8 @@ ENV BROKER_PASSWORD="fda" ENV SEARCH="http://search-db:9200" ENV SHARED_FILESYSTEM="/tmp" ENV LOGO="/logo.png" -ENV ELASTIC_USERNAME="elastic" -ENV ELASTIC_PASSWORD="elastic" +ENV SEARCH_USERNAME="admin" +ENV SEARCH_PASSWORD="admin" ENV VERSION="${TAG}" ENV TITLE="Database Repository" ENV ICON="/favicon.ico" diff --git a/dbrepo-ui/config.js b/dbrepo-ui/config.js index b12b7b7055b3bc0b08422bcd385093de08fdc742..e83a19c0c0a0cf7caf9d96c6b8709a9e9d9756bf 100644 --- a/dbrepo-ui/config.js +++ b/dbrepo-ui/config.js @@ -12,7 +12,7 @@ config.version = process.env.VERSION || 'latest' config.logo = process.env.LOGO || '/logo.png' config.mailVerify = process.env.MAIL_VERIFY || false config.tokenMax = process.env.TOKEN_MAX || 5 -config.elasticPassword = process.env.ELASTIC_PASSWORD || 'elastic' +config.elasticPassword = process.env.SEARCH_PASSWORD || 'elastic' config.clientId = process.env.DBREPO_CLIENT_ID || 'dbrepo-client' config.clientSecret = process.env.DBREPO_CLIENT_SECRET || 'MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG' config.defaultPublisher = process.env.DEFAULT_PID_PUBLISHER || 'Example University' diff --git a/dbrepo-user-service/Dockerfile b/dbrepo-user-service/Dockerfile index fb50823d3c479dacb13530a0b9efed658d521c59..b3c9c5faa1b75aceafebf8a146db78fd874d84f5 100644 --- a/dbrepo-user-service/Dockerfile +++ b/dbrepo-user-service/Dockerfile @@ -33,7 +33,7 @@ ENV JWT_ISSUER="http://localhost/realms/dbrepo" ENV JWT_PUBKEY="MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB" ENV LOG_LEVEL=debug ENV DEFAULT_ROLE="default-researcher-roles" -ENV ELASTIC_PASSWORD=elastic +ENV SEARCH_PASSWORD=admin WORKDIR /app diff --git a/dbrepo-user-service/pom.xml b/dbrepo-user-service/pom.xml index 30e36bcba0c122d7376c44e8cd372668f0294f72..dad7c33127b212198300dfd352c13deb04495e6f 100644 --- a/dbrepo-user-service/pom.xml +++ b/dbrepo-user-service/pom.xml @@ -17,7 +17,7 @@ also allows for view-only (possibly paginated and versioned) query execution to the raw data and consumes messages in the message queue from the Broker Service. </description> - <url>https://dbrepo-docs.ossdip.at</url> + <url>https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/</url> <developers> <developer> <name>Martin Weise</name> @@ -52,6 +52,7 @@ <keycloak.version>21.0.2</keycloak.version> <hibernate.version>6.2.2.Final</hibernate.version> <springdoc-openapi.version>2.1.0</springdoc-openapi.version> + <opensearch.version>1.1.0</opensearch.version> </properties> <dependencies> @@ -116,8 +117,9 @@ <version>${mariadb.version}</version> </dependency> <dependency> - <groupId>org.springframework.data</groupId> - <artifactId>spring-data-elasticsearch</artifactId> + <groupId>org.opensearch.client</groupId> + <artifactId>spring-data-opensearch-starter</artifactId> + <version>${opensearch.version}</version> </dependency> <!-- Swagger --> <dependency> diff --git a/dbrepo-user-service/rest-service/src/main/resources/application-local.yml b/dbrepo-user-service/rest-service/src/main/resources/application-local.yml index e15f94b2cc50f9c62e045667cb5940994a29a7cb..c0c994b5ff4a48a41a81733e7061092c43c52601 100644 --- a/dbrepo-user-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-user-service/rest-service/src/main/resources/application-local.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: fda password: fda - elasticsearch: - password: elastic - username: elastic + opensearch: + username: admin + password: admin uris: http://localhost:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-user-service/rest-service/src/main/resources/application.yml b/dbrepo-user-service/rest-service/src/main/resources/application.yml index 58ca1c7f57132a226ee7437ab48e3345e011becf..910db255f153dbfc1c9769c9b1fbf8072ebc0d7a 100644 --- a/dbrepo-user-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-user-service/rest-service/src/main/resources/application.yml @@ -25,9 +25,9 @@ spring: virtual-host: dbrepo username: "${BROKER_USERNAME}" password: "${BROKER_PASSWORD}" - elasticsearch: - password: "${ELASTIC_PASSWORD}" - username: elastic + opensearch: + username: "${SEARCH_USERNAME}" + password: "${SEARCH_PASSWORD}" uris: http://search-db:9200 cloud: loadbalancer.ribbon.enabled: false diff --git a/dbrepo-user-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/dbrepo-user-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java deleted file mode 100644 index 58079c0b9c7c08d8c0ae75ee1ecf0191a7d19f65..0000000000000000000000000000000000000000 --- a/dbrepo-user-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -package at.tuwien.config; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; -import lombok.extern.log4j.Log4j2; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.elasticsearch.client.RestClient; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Log4j2 -@Configuration -public class ElasticsearchConfig { - - @Value("${spring.elasticsearch.uris}") - private String elasticEndpoint; - - @Value("${spring.elasticsearch.username}") - private String elasticUsername; - - @Value("${spring.elasticsearch.password}") - private String elasticPassword; - - @Bean - public ElasticsearchClient elasticsearchClient() { - log.debug("elastic endpoint={}", elasticEndpoint); - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword)); - final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint)) - .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider)) - .build(); - ElasticsearchTransport transport = new RestClientTransport( - restClient, new JacksonJsonpMapper()); - return new ElasticsearchClient(transport); - } - -} \ No newline at end of file diff --git a/dbrepo-user-service/services/src/main/java/at/tuwien/config/IndexConfig.java b/dbrepo-user-service/services/src/main/java/at/tuwien/config/IndexConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..ba3a4c634edd0cbf4556683ac92c29f7087e4e79 --- /dev/null +++ b/dbrepo-user-service/services/src/main/java/at/tuwien/config/IndexConfig.java @@ -0,0 +1,53 @@ +package at.tuwien.config; + +import at.tuwien.api.user.UserDto; +import at.tuwien.mapper.UserMapper; +import at.tuwien.repository.elastic.UserIdxRepository; +import at.tuwien.repository.jpa.UserRepository; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +@Component +@Log4j2 +public class IndexConfig { + + private final UserMapper userMapper; + private final UserRepository userRepository; + private final UserIdxRepository userIdxRepository; + private final ElasticsearchOperations elasticsearchOperations; + + @Autowired + public IndexConfig(UserMapper userMapper, UserRepository userRepository, + UserIdxRepository userIdxRepository, ElasticsearchOperations elasticsearchOperations) { + this.userMapper = userMapper; + this.userRepository = userRepository; + this.userIdxRepository = userIdxRepository; + this.elasticsearchOperations = elasticsearchOperations; + } + + @Transactional + @EventListener(ApplicationReadyEvent.class) + public void initIndex() { + final IndexCoordinates userIndex = IndexCoordinates.of("user"); + if (!elasticsearchOperations.indexOps(userIndex).exists()) { + elasticsearchOperations.indexOps(userIndex).create(); + elasticsearchOperations.indexOps(userIndex).createMapping(UserDto.class); + log.info("Created user index"); + } + /* pre-fill */ + final List<UserDto> users = userRepository.findAll() + .stream() + .map(userMapper::userToUserDto) + .toList(); + userIdxRepository.saveAll(users); + log.info("Added {} users to OpenSearch index", users.size()); + } +} diff --git a/dbrepo-user-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java b/dbrepo-user-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..27d815cc6f4329c5374c31fb5299ccabedf24d39 --- /dev/null +++ b/dbrepo-user-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.client.RestHighLevelClient; +import org.opensearch.data.client.orhlc.ClientConfiguration; +import org.opensearch.data.client.orhlc.RestClients; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Log4j2 +@Configuration +public class OpenSearchConfig { + + @Value("${spring.opensearch.uris}") + private String openSearchEndpoint; + + @Value("${spring.opensearch.username}") + private String openSearchUsername; + + @Value("${spring.opensearch.password}") + private String openSearchPassword; + + @Bean + public RestHighLevelClient openSearchClient() { + log.debug("openSearch endpoint={}", openSearchEndpoint); + final ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(openSearchEndpoint) + .withBasicAuth(openSearchUsername, openSearchPassword) + .build(); + return RestClients.create(clientConfiguration) + .rest(); + } + +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 909b52556b9465f3e8300ea92fc54fb09bf39fe9..facea5fad7f8e1b1e1d5a130b31899b0663324bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -296,7 +296,7 @@ services: restart: always container_name: dbrepo-search-db hostname: search-db - image: elasticsearch:8.7.1 + image: opensearchproject/opensearch:2 networks: core: ports: @@ -305,10 +305,14 @@ services: - .env environment: discovery.type: "single-node" - ES_JAVA_OPTS: "-Xms2g -Xmx2g" + ES_JAVA_OPTS: "-Xms4g -Xmx4g" logger.level: "WARN" + plugins.security.disabled: "true" bootstrap.memory_lock: "true" - xpack.security.enabled: "true" + deploy: + resources: + limits: + memory: 4G volumes: - search-db-data:/usr/share/elasticsearch/data logging: