WIP opensearch

.env.unix.example

@@ -4,7 +4,8 @@ JWT_ISSUER="http://localhost/realms/dbrepo"
 JWT_PUBKEY="MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB"
 JWT_CERT="MIICmzCCAYMCBgGG3GWyBTANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZkYnJlcG8wHhcNMjMwMzEzMTkxMzE3WhcNMzMwMzEzMTkxNDU3WjARMQ8wDQYDVQQDDAZkYnJlcG8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCqqcdDYFZZb28M0tEJzEP77FmD/Xqioyj9zWX6VwUSOMAgmMmn8eqs9hT9T0a+q4YTo9tUW1PNbUpwprA5b4Uk04DcIajxDVMUR/PjcHytmkqwVskq9AZW/Vngdoo+8tSbuIybwe/3Vwt266hbHpDcM97a+DXcYooRl7tQWCEX7RP27wQrMD9epDQ6IgKayZg9vC9/03dsIqwH9jXQRiZlFvwiEKhX2aY7lPGBaCK414JO00K/Z49iov9TRa/IYVbSt5qwgrx6DcqsBSPwOnI6A85UGfeUEZ/7coVJiL7RvBlsllapsL9eWTbQajVh94k9Ei3sibEPbtH+U2OAM78zAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAASnN1Cuif1sdfEK2kWAURSXGJCohCROLWdKFjaeHPRaEfpbFJsgxW0Yj3nwX5O3bUlOWoTyENwnXSsXMQsqnNi+At32CKaKO8+AkhAbgQL9F0B+KeJwmYv3cUj5N/LYkJjBvZBzUZ4Ugu5dcxH0k7AktLAIwimkyEnxTNolOA3UyrGGpREr8MCKWVr10RFuOpF/0CsJNNwbHXzalO9D756EUcRWZ9VSg6QVNso0YYRKTnILWDn9hcTRnqGy3SHo3anFTqQZ+BB57YbgFWy6udC0LYRB3zdp6zNti87eu/VEymiDY/mmo1AB8Tm0b6vxFz4AKcL3ax5qS6YnZ9efSzk="
 SHARED_FILESYSTEM=/tmp
-ELASTIC_PASSWORD=elastic
+SEARCH_USERNAME=admin
+SEARCH_PASSWORD=admin
 METADATA_DB=fda
 METADATA_USERNAME=root
 METADATA_PASSWORD=dbrepo

README.md

@@ -40,7 +40,7 @@ $ docker-compose build --parallel
 
 A more detailed description on how
 to get started is available at our documentation
-website: [https://dbrepo-docs.ossdip.at/getting-started/](https://dbrepo-docs.ossdip.at/getting-started/)
+website: [https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//getting-started/](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//getting-started/)
 
 ### Run
 

dbrepo-container-service/Dockerfile

@@ -30,8 +30,8 @@ ENV METADATA_USERNAME=root
 ENV METADATA_PASSWORD=dbrepo
 ENV BROKER_USERNAME=fda
 ENV BROKER_PASSWORD=fda
-ENV ELASTIC_USERNAME=elastic
-ENV ELASTIC_PASSWORD=elastic
+ENV SEARCH_USERNAME=admin
+ENV SEARCH_PASSWORD=admin
 ENV SHARED_FILESYSTEM=/tmp
 ENV USER_NETWORK=userdb
 ENV LOG_LEVEL=debug

dbrepo-container-service/pom.xml

@@ -30,6 +30,7 @@
         <jacoco.version>0.8.10</jacoco.version>
         <jwt.version>4.3.0</jwt.version>
         <springdoc-openapi.version>2.1.0</springdoc-openapi.version>
+        <opensearch-client.version>1.1.0</opensearch-client.version>
     </properties>
 
     <dependencies>
@@ -88,6 +89,16 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-jpa</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.opensearch.client</groupId>
+            <artifactId>spring-data-opensearch</artifactId>
+            <version>${opensearch-client.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.opensearch.client</groupId>
+            <artifactId>spring-data-opensearch-starter</artifactId>
+            <version>${opensearch-client.version}</version>
+        </dependency>
         <!-- Entity and API -->
         <dependency>
             <groupId>at.tuwien</groupId>
@@ -108,6 +119,12 @@
             <version>${mariadb.version}</version>
         </dependency>
         <!-- Testing -->
+        <dependency>
+            <groupId>org.opensearch.client</groupId>
+            <artifactId>spring-data-opensearch-test-autoconfigure</artifactId>
+            <version>${opensearch-client.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>

dbrepo-container-service/rest-service/src/main/resources/application-local.yml

@@ -27,10 +27,6 @@ spring:
     virtual-host: dbrepo
     username: fda
     password: fda
-  elasticsearch:
-    password: elastic
-    username: elastic
-    uris: http://localhost:9200
 management.endpoints.web.exposure.include: health,info,prometheus
 springdoc.swagger-ui.enabled: true
 server:
@@ -51,3 +47,7 @@ fda:
   client_secret: client-secret
   client_id: dbrepo-client
   gateway.endpoint: http://localhost
+  opensearch:
+    username: admin
+    password: admin
+    uris: http://localhost:9200
\ No newline at end of file

dbrepo-container-service/rest-service/src/main/resources/application.yml

@@ -27,10 +27,6 @@ spring:
     virtual-host: dbrepo
     username: "${BROKER_USERNAME}"
     password: "${BROKER_PASSWORD}"
-  elasticsearch:
-    password: "${ELASTIC_PASSWORD}"
-    username: "${ELASTIC_USERNAME}"
-    uris: http://search-db:9200
 management.endpoints.web.exposure.include: health,info,prometheus
 springdoc.swagger-ui.enabled: true
 server:
@@ -51,3 +47,7 @@ fda:
   client_secret: "${DBREPO_CLIENT_SECRET}"
   client_id: "${CLIENT_ID}"
   gateway.endpoint: http://gateway-service
+  opensearch:
+    username: "${SEARCH_USERNAME}"
+    password: "${SEARCH_PASSWORD}"
+    uris: http://search-db:9200
\ No newline at end of file

dbrepo-database-service/Dockerfile

@@ -30,8 +30,8 @@ ENV METADATA_USERNAME=root
 ENV METADATA_PASSWORD=dbrepo
 ENV BROKER_USERNAME=fda
 ENV BROKER_PASSWORD=fda
-ENV ELASTIC_USERNAME=elastic
-ENV ELASTIC_PASSWORD=elastic
+ENV SEARCH_USERNAME=admin
+ENV SEARCH_PASSWORD=admin
 ENV GATEWAY_ENDPOINT=http://gateway-service
 ENV LOG_LEVEL=debug
 ENV CLIENT_ID="dbrepo-client"

dbrepo-database-service/pom.xml

@@ -31,6 +31,7 @@
         <c3p0.version>0.9.5.5</c3p0.version>
         <c3p0-hibernate.version>6.2.2.Final</c3p0-hibernate.version>
         <springdoc-openapi.version>2.1.0</springdoc-openapi.version>
+        <opensearch.version>1.1.0</opensearch.version>
     </properties>
 
     <dependencies>
@@ -69,9 +70,9 @@
         </dependency>
         <!-- elasticsearch -->
         <dependency>
-            <groupId>co.elastic.clients</groupId>
-            <artifactId>elasticsearch-java</artifactId>
-            <version>${elasticsearch-client.version}</version>
+            <groupId>org.opensearch.client</groupId>
+            <artifactId>spring-data-opensearch-starter</artifactId>
+            <version>${opensearch.version}</version>
         </dependency>
         <!-- AMQP -->
         <dependency>

dbrepo-database-service/rest-service/src/main/resources/application.yml

@@ -27,9 +27,9 @@ spring:
     virtual-host: dbrepo
     username: "${BROKER_USERNAME}"
     password: "${BROKER_PASSWORD}"
-  elasticsearch:
-    password: "${ELASTIC_PASSWORD}"
-    username: "${ELASTIC_USERNAME}"
+  opensearch:
+    password: "${SEARCH_PASSWORD}"
+    username: "${SEARCH_USERNAME}"
     uris: http://search-db:9200
 management.endpoints.web.exposure.include: health,info,prometheus
 server:

dbrepo-database-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java

-package at.tuwien.config;
-
-import co.elastic.clients.elasticsearch.ElasticsearchClient;
-import co.elastic.clients.json.jackson.JacksonJsonpMapper;
-import co.elastic.clients.transport.ElasticsearchTransport;
-import co.elastic.clients.transport.rest_client.RestClientTransport;
-import lombok.extern.log4j.Log4j2;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.elasticsearch.client.RestClient;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
-
-@Log4j2
-@Configuration
-public class ElasticsearchConfig {
-
-    @Value("${spring.elasticsearch.uris}")
-    private String elasticEndpoint;
-
-    @Value("${spring.elasticsearch.username}")
-    private String elasticUsername;
-
-    @Value("${spring.elasticsearch.password}")
-    private String elasticPassword;
-
-    @Bean
-    public ElasticsearchClient elasticsearchClient() {
-        log.debug("elastic endpoint={}", elasticEndpoint);
-        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(elasticUsername, elasticPassword));
-        final RestClient restClient = RestClient.builder(HttpHost.create(elasticEndpoint))
-                .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder
-                        .setDefaultCredentialsProvider(credentialsProvider))
-                .build();
-        ElasticsearchTransport transport = new RestClientTransport(
-                restClient, new JacksonJsonpMapper());
-        return new ElasticsearchClient(transport);
-    }
-
-}
\ No newline at end of file

dbrepo-database-service/services/src/main/java/at/tuwien/config/IndexConfig.java

@@ -21,38 +21,33 @@ import java.util.stream.Collectors;
 @Component
 public class IndexConfig {
 
-    private final Environment environment;
     private final DatabaseMapper databaseMapper;
     private final DatabaseRepository databaseRepository;
     private final DatabaseIdxRepository databaseIdxRepository;
     private final ElasticsearchOperations elasticsearchOperations;
-    private final ApplicationEventPublisher applicationEventPublisher;
 
-    public IndexConfig(Environment environment, DatabaseMapper databaseMapper, DatabaseRepository databaseRepository,
-                       DatabaseIdxRepository databaseIdxRepository, ElasticsearchOperations elasticsearchOperations,
-                       ApplicationEventPublisher applicationEventPublisher) {
-        this.environment = environment;
+    public IndexConfig(DatabaseMapper databaseMapper, DatabaseRepository databaseRepository,
+                       DatabaseIdxRepository databaseIdxRepository, ElasticsearchOperations elasticsearchOperations) {
         this.databaseMapper = databaseMapper;
         this.databaseRepository = databaseRepository;
         this.databaseIdxRepository = databaseIdxRepository;
         this.elasticsearchOperations = elasticsearchOperations;
-        this.applicationEventPublisher = applicationEventPublisher;
     }
 
     @Transactional
     @EventListener(ApplicationReadyEvent.class)
     public void initIndex() {
-        log.debug("creating databaseindex");
-        final IndexCoordinates databaseIndex = IndexCoordinates.of("databaseindex");
+        final IndexCoordinates databaseIndex = IndexCoordinates.of("database");
         if (!elasticsearchOperations.indexOps(databaseIndex).exists()) {
             elasticsearchOperations.indexOps(databaseIndex).create();
             elasticsearchOperations.indexOps(databaseIndex).createMapping(DatabaseDto.class);
+            log.info("Created identifier index");
         }
         final List<DatabaseDto> databases = databaseRepository.findAll()
                 .stream()
                 .map(databaseMapper::databaseToDatabaseDto)
                 .collect(Collectors.toList());
-        log.debug("add {} databases to elastic search index", databases.size());
+        log.info("Added {} databases to OpenSearch index", databases.size());
         databaseIdxRepository.saveAll(databases);
     }
 

dbrepo-database-service/services/src/main/java/at/tuwien/config/OpenSearchConfig.java

+package at.tuwien.config;
+
+import lombok.extern.log4j.Log4j2;
+import org.opensearch.client.RestHighLevelClient;
+import org.opensearch.data.client.orhlc.ClientConfiguration;
+import org.opensearch.data.client.orhlc.RestClients;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Log4j2
+@Configuration
+public class OpenSearchConfig {
+
+    @Value("${spring.opensearch.uris}")
+    private String openSearchEndpoint;
+
+    @Value("${spring.opensearch.username}")
+    private String openSearchUsername;
+
+    @Value("${spring.opensearch.password}")
+    private String openSearchPassword;
+
+    @Bean
+    public RestHighLevelClient openSearchClient() {
+        log.debug("openSearch endpoint={}", openSearchEndpoint);
+        final ClientConfiguration clientConfiguration = ClientConfiguration.builder()
+                .connectedTo(openSearchEndpoint)
+                .withBasicAuth(openSearchUsername, openSearchPassword)
+                .build();
+        return RestClients.create(clientConfiguration)
+                .rest();
+    }
+
+}
\ No newline at end of file

dbrepo-identifier-service/pom.xml

@@ -35,6 +35,7 @@
         <super-csv.version>2.4.0</super-csv.version>
         <jsql.version>4.6</jsql.version>
         <springdoc-openapi.version>2.1.0</springdoc-openapi.version>
+        <opensearch.version>1.1.0</opensearch.version>
     </properties>
 
     <dependencies>
@@ -98,8 +99,9 @@
             <version>${mariadb.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.springframework.data</groupId>
-            <artifactId>spring-data-elasticsearch</artifactId>
+            <groupId>org.opensearch.client</groupId>
+            <artifactId>spring-data-opensearch-starter</artifactId>
+            <version>${opensearch.version}</version>
         </dependency>
         <!-- Docker -->
         <dependency>

dbrepo-identifier-service/rest-service/src/main/resources/application-local.yml

@@ -27,9 +27,9 @@ spring:
     virtual-host: dbrepo
     username: fda
     password: fda
-  elasticsearch:
-    password: elastic
-    username: elastic
+  opensearch:
+    username: admin
+    password: admin
     uris: http://localhost:9200
 management.endpoints.web.exposure.include: health,info,prometheus
 server:

dbrepo-identifier-service/rest-service/src/main/resources/application.yml

@@ -27,9 +27,9 @@ spring:
     virtual-host: dbrepo
     username: "${BROKER_USERNAME}"
     password: "${BROKER_PASSWORD}"
-  elasticsearch:
-    password: "${ELASTIC_PASSWORD}"
-    username: elastic
+  opensearch:
+    username: "${SEARCH_USERNAME}"
+    password: "${SEARCH_PASSWORD}"
     uris: http://search-db:9200
 management.endpoints.web.exposure.include: health,info,prometheus
 server:

dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/ActuatorComponentTest.java

 package at.tuwien.endpoint;
 
 import at.tuwien.BaseUnitTest;
-import at.tuwien.config.IndexInitializer;
+import at.tuwien.config.IndexConfig;
 import lombok.extern.log4j.Log4j2;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -26,7 +26,7 @@ public class ActuatorComponentTest extends BaseUnitTest {
     private MockMvc mockMvc;
 
     @MockBean
-    private IndexInitializer indexConfig;
+    private IndexConfig indexConfig;
 
     @Test
     public void actuatorInfo_succeeds() throws Exception {

dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointIntegrationTest.java

@@ -3,7 +3,7 @@ package at.tuwien.endpoint;
 import at.tuwien.BaseUnitTest;
 import at.tuwien.api.identifier.IdentifierDto;
 import at.tuwien.api.identifier.IdentifierTypeDto;
-import at.tuwien.config.IndexInitializer;
+import at.tuwien.config.IndexConfig;
 import at.tuwien.config.ReadyConfig;
 import at.tuwien.endpoints.IdentifierEndpoint;
 import at.tuwien.exception.NotAllowedException;
@@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.*;
 public class IdentifierEndpointIntegrationTest extends BaseUnitTest {
 
     @MockBean
-    private IndexInitializer indexInitializer;
+    private IndexConfig indexInitializer;
 
     @MockBean
     private ReadyConfig readyConfig;

dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/IdentifierEndpointUnitTest.java

@@ -5,7 +5,7 @@ import at.tuwien.api.identifier.IdentifierCreateDto;
 import at.tuwien.api.identifier.IdentifierDto;
 import at.tuwien.api.identifier.IdentifierTypeDto;
 import at.tuwien.config.EndpointConfig;
-import at.tuwien.config.IndexInitializer;
+import at.tuwien.config.IndexConfig;
 import at.tuwien.config.ReadyConfig;
 import at.tuwien.endpoints.IdentifierEndpoint;
 import at.tuwien.endpoints.PersistenceEndpoint;
@@ -29,7 +29,6 @@ import org.springframework.core.io.InputStreamResource;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.AccessDeniedException;
-import org.springframework.security.authentication.AuthenticationCredentialsNotFoundException;
 import org.springframework.security.test.context.support.WithAnonymousUser;
 import org.springframework.security.test.context.support.WithMockUser;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
@@ -54,7 +53,7 @@ public class IdentifierEndpointUnitTest extends BaseUnitTest {
     private ReadyConfig readyConfig;
 
     @MockBean
-    private IndexInitializer indexInitializer;
+    private IndexConfig indexInitializer;
 
     @MockBean
     private IdentifierService identifierService;

dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/PersistenceEndpointUnitTest.java

@@ -5,7 +5,7 @@ import at.tuwien.api.identifier.BibliographyTypeDto;
 import at.tuwien.api.identifier.CreatorDto;
 import at.tuwien.api.identifier.IdentifierDto;
 import at.tuwien.api.identifier.IdentifierUpdateDto;
-import at.tuwien.config.IndexInitializer;
+import at.tuwien.config.IndexConfig;
 import at.tuwien.config.ReadyConfig;
 import at.tuwien.endpoints.PersistenceEndpoint;
 import at.tuwien.entities.identifier.Identifier;
@@ -54,7 +54,7 @@ public class PersistenceEndpointUnitTest extends BaseUnitTest {
     private ReadyConfig readyConfig;
 
     @MockBean
-    private IndexInitializer indexInitializer;
+    private IndexConfig indexInitializer;
 
     @MockBean
     private AccessService accessService;

dbrepo-identifier-service/rest-service/src/test/java/at/tuwien/endpoint/SwaggerComponentTest.java

 package at.tuwien.endpoint;
 
 import at.tuwien.BaseUnitTest;
-import at.tuwien.config.IndexInitializer;
+import at.tuwien.config.IndexConfig;
 import lombok.extern.log4j.Log4j2;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -26,7 +26,7 @@ public class SwaggerComponentTest extends BaseUnitTest {
     private MockMvc mockMvc;
 
     @MockBean
-    private IndexInitializer indexConfig;
+    private IndexConfig indexConfig;
 
     @Test
     public void swaggerUi_succeeds() throws Exception {