diff --git a/docker-compose.yml b/docker-compose.yml index bc416d058235daa06aa3ae9e59033d734af46db7..0db075af6ce3bccd46a9daee77c1625632be4572 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -230,6 +230,7 @@ services: container_name: fda-search-service hostname: fda-search-service image: elasticsearch:7.13.4 + command: ["elasticsearch"] networks: - fda-public environment: @@ -239,6 +240,8 @@ services: ports: - 9200:9200 - 9600:9600 + logging: + driver: json-file fda-ui: restart: on-failure @@ -269,4 +272,4 @@ services: API_IMAGE: http://fda-container-service:9091 API_DATABASE: http://fda-database-service:9092 API_TABLES: http://fda-table-service:9094 - API_ANALYSE: http://fda-analyse-service:5000 \ No newline at end of file + API_ANALYSE: http://fda-analyse-service:5000 diff --git a/fda-authentication-service/.gitignore b/fda-authentication-service/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..862815a0c53788c70306117181f3ab764b56597b --- /dev/null +++ b/fda-authentication-service/.gitignore @@ -0,0 +1,2 @@ +ready +*.key diff --git a/fda-container-service/rest-service/src/test/java/at/tuwien/endpoint/ImageEndpointUnitTest.java b/fda-container-service/rest-service/src/test/java/at/tuwien/endpoint/ImageEndpointUnitTest.java index 4a9a960eea00a0f7c0a8a1dd3facd15d362a594d..21ce00e213c668fbc64e78aec8a8645eea495310 100644 --- a/fda-container-service/rest-service/src/test/java/at/tuwien/endpoint/ImageEndpointUnitTest.java +++ b/fda-container-service/rest-service/src/test/java/at/tuwien/endpoint/ImageEndpointUnitTest.java @@ -114,7 +114,6 @@ public class ImageEndpointUnitTest extends BaseUnitTest { /* test */ final ResponseEntity<ImageDto> response = imageEndpoint.findById(IMAGE_1_ID); assertEquals(HttpStatus.OK, response.getStatusCode()); - assertEquals(IMAGE_1_HASH, Objects.requireNonNull(response.getBody()).getHash()); } @Test diff --git a/fda-database-service/rest-service/src/test/java/at/tuwien/endpoint/EndpointUnitTest.java b/fda-database-service/rest-service/src/test/java/at/tuwien/endpoint/EndpointUnitTest.java index 0ae40566d4e8ee853fa29950a3eb172c26b98b29..8fc3b0c5734adb601ceca9761af50ad92ad704a1 100644 --- a/fda-database-service/rest-service/src/test/java/at/tuwien/endpoint/EndpointUnitTest.java +++ b/fda-database-service/rest-service/src/test/java/at/tuwien/endpoint/EndpointUnitTest.java @@ -50,6 +50,7 @@ public class EndpointUnitTest extends BaseUnitTest { @BeforeAll public static void beforeAll() throws InterruptedException { + afterAll(); final DockerConfig dockerConfig = new DockerConfig(); final HostConfig hostConfig = dockerConfig.hostConfig(); final DockerClient dockerClient = dockerConfig.dockerClientConfiguration(); diff --git a/fda-database-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java b/fda-database-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java index bc00049b1445da3a77e6c133ddfdff7432bd16a0..bfb87d7eba150f1a646293cf92bc66abaa538bb7 100644 --- a/fda-database-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java +++ b/fda-database-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java @@ -8,6 +8,7 @@ import at.tuwien.config.ReadyConfig; import at.tuwien.entities.container.Container; import at.tuwien.entities.database.Database; import at.tuwien.exception.*; +import at.tuwien.repository.elastic.DatabaseidxRepository; import at.tuwien.repository.jpa.ContainerRepository; import at.tuwien.repository.jpa.DatabaseRepository; import at.tuwien.repository.jpa.ImageRepository; @@ -47,6 +48,9 @@ public class DatabaseServiceIntegrationTest extends BaseUnitTest { @MockBean private Channel channel; + @MockBean + private DatabaseidxRepository databaseidxRepository; + @Autowired private HostConfig hostConfig; @@ -145,6 +149,7 @@ public class DatabaseServiceIntegrationTest extends BaseUnitTest { containerRepository.save(CONTAINER_2); } + @Transactional @AfterEach public void afterEach() { /* stop containers and remove them */ diff --git a/fda-database-service/rest-service/src/test/resources/application.properties b/fda-database-service/rest-service/src/test/resources/application.properties index 1f8a9f4089f393b2a8d25f28472a80a35658aab5..c62372cb85f685babd35737f4b0e26aa8d7a1dcc 100644 --- a/fda-database-service/rest-service/src/test/resources/application.properties +++ b/fda-database-service/rest-service/src/test/resources/application.properties @@ -1,3 +1,5 @@ +spring.profiles.active=test-noelastic + # disable discovery spring.cloud.discovery.enabled = false @@ -12,4 +14,7 @@ spring.datasource.username=sa spring.datasource.password=password spring.jpa.database-platform=org.hibernate.dialect.H2Dialect spring.jpa.hibernate.ddl-auto=create-drop -spring.jpa.show-sql=false \ No newline at end of file +spring.jpa.show-sql=false + +# disable elasticsearch +spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration,org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchRepositoriesAutoConfiguration,org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration \ No newline at end of file diff --git a/fda-database-service/services/src/main/java/at/tuwien/config/IndexInitializer.java b/fda-database-service/services/src/main/java/at/tuwien/config/IndexInitializer.java new file mode 100644 index 0000000000000000000000000000000000000000..5c6544af42f09c3f6809edaeefd9d6b02f2fd91e --- /dev/null +++ b/fda-database-service/services/src/main/java/at/tuwien/config/IndexInitializer.java @@ -0,0 +1,37 @@ +package at.tuwien.config; + +import at.tuwien.entities.database.Database; +import lombok.extern.log4j.Log4j2; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.env.Environment; +import org.springframework.core.env.Profiles; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.stereotype.Component; + +@Component +@Log4j2 +public class IndexInitializer { + + private final Environment environment; + private final ElasticsearchOperations elasticsearchOperations; + + public IndexInitializer(Environment environment, ElasticsearchOperations elasticsearchOperations) { + this.environment = environment; + this.elasticsearchOperations = elasticsearchOperations; + } + + @EventListener(ApplicationReadyEvent.class) + public void initIndex() { + if (environment.acceptsProfiles(Profiles.of("test-noelastic"))) { + return; + } + log.debug("creating index"); + IndexCoordinates indexCoordinates = IndexCoordinates.of("databaseindex"); + if (!elasticsearchOperations.indexOps(indexCoordinates).exists()) { + elasticsearchOperations.indexOps(indexCoordinates).create(); + elasticsearchOperations.indexOps(indexCoordinates).createMapping(Database.class); + } + } +} diff --git a/fda-database-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseRepository.java b/fda-database-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseidxRepository.java similarity index 75% rename from fda-database-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseRepository.java rename to fda-database-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseidxRepository.java index e02e132395c27e3455846305df1cdb6505722d77..629d38c73ef5fa89b4ed3e1bc9ca9e5ba621980a 100644 --- a/fda-database-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseRepository.java +++ b/fda-database-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseidxRepository.java @@ -5,5 +5,5 @@ import org.springframework.data.elasticsearch.repository.ElasticsearchRepository import org.springframework.stereotype.Repository; @Repository(value = "ElasticDatabaseRepository") -public interface DatabaseRepository extends ElasticsearchRepository<Database, Long> { +public interface DatabaseidxRepository extends ElasticsearchRepository<Database, Long> { } diff --git a/fda-database-service/services/src/main/java/at/tuwien/service/DatabaseService.java b/fda-database-service/services/src/main/java/at/tuwien/service/DatabaseService.java index cfbaae59fda42025a1d3fbc9dfca63b29d4707e0..042606a77ee331efd4fa3cbd633a8e4a1418d2af 100644 --- a/fda-database-service/services/src/main/java/at/tuwien/service/DatabaseService.java +++ b/fda-database-service/services/src/main/java/at/tuwien/service/DatabaseService.java @@ -11,6 +11,7 @@ import at.tuwien.mapper.DatabaseMapper; import at.tuwien.mapper.ImageMapper; import at.tuwien.repository.jpa.ContainerRepository; import at.tuwien.repository.jpa.DatabaseRepository; +import at.tuwien.repository.elastic.DatabaseidxRepository; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -31,18 +32,21 @@ public class DatabaseService extends JdbcConnector { private final ContainerRepository containerRepository; private final DatabaseRepository databaseRepository; + private final DatabaseidxRepository databaseidxRepository; private final DatabaseMapper databaseMapper; private final AmqpService amqpService; private final AmqpMapper amqpMapper; @Autowired public DatabaseService(ContainerRepository containerRepository, DatabaseRepository databaseRepository, - ImageMapper imageMapper, DatabaseMapper databaseMapper, + DatabaseidxRepository databaseidxRepository, ImageMapper imageMapper, + DatabaseMapper databaseMapper, AmqpService amqpService, AmqpMapper amqpMapper) { super(imageMapper, databaseMapper); this.containerRepository = containerRepository; this.databaseRepository = databaseRepository; this.databaseMapper = databaseMapper; + this.databaseidxRepository = databaseidxRepository; this.amqpService = amqpService; this.amqpMapper = amqpMapper; } @@ -125,6 +129,8 @@ public class DatabaseService extends JdbcConnector { final Database out = databaseRepository.save(database); log.info("Created database {}", out.getId()); log.debug("created database {}", out); + // save in database_index - elastic search + databaseidxRepository.save(database); return out; } @@ -148,6 +154,8 @@ public class DatabaseService extends JdbcConnector { final Database out = databaseRepository.save(database); log.info("Updated database {}", out.getId()); log.debug("updated database {}", out); + // save in database_index - elastic search + databaseidxRepository.save(database); return out; } diff --git a/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/Database.java b/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/Database.java index 3e677c784446fc72b02cbfd1d561dd540d8ffb66..6c2f22431f5f844de6059ef9dbc0c81be8e0441a 100644 --- a/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/Database.java +++ b/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/Database.java @@ -21,8 +21,8 @@ import java.util.List; @Builder @AllArgsConstructor @NoArgsConstructor +@Document(indexName = "databaseindex", createIndex = false) @Where(clause = "deleted is null") -@Document(indexName = "databaseindex") @ToString(onlyExplicitlyIncluded = true) @EntityListeners(AuditingEntityListener.class) @EqualsAndHashCode(onlyExplicitlyIncluded = true) diff --git a/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/table/Table.java b/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/table/Table.java index c8b102b49277746e228d0c707c05f2b358fb9fb6..94b5da4c9d7464383f9d7995bbbe06f353828e7f 100644 --- a/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/table/Table.java +++ b/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/table/Table.java @@ -6,6 +6,9 @@ import lombok.*; import org.hibernate.annotations.GenericGenerator; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.elasticsearch.annotations.Document; +import org.springframework.data.elasticsearch.annotations.Field; +import org.springframework.data.elasticsearch.annotations.FieldType; import org.springframework.data.jpa.domain.support.AuditingEntityListener; import javax.persistence.*; @@ -17,6 +20,7 @@ import java.util.List; @Builder @AllArgsConstructor @NoArgsConstructor +@Document(indexName = "tblindex", createIndex = false) @IdClass(TableKey.class) @ToString(onlyExplicitlyIncluded = true) @EntityListeners(AuditingEntityListener.class) @@ -67,6 +71,7 @@ public class Table { @ToString.Include @OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL, mappedBy = "table") + @Field(type = FieldType.Nested) private List<TableColumn> columns; @Column(nullable = false, updatable = false) diff --git a/fda-table-service/rest-service/src/main/resources/application-docker.yml b/fda-table-service/rest-service/src/main/resources/application-docker.yml index b0cdc68c9a5a311b583922580576767398e4a22d..3af598b5a002fb0ccc62e37e7e5f09abfeae7ef5 100644 --- a/fda-table-service/rest-service/src/main/resources/application-docker.yml +++ b/fda-table-service/rest-service/src/main/resources/application-docker.yml @@ -31,6 +31,7 @@ eureka: fda: mapping.path: /root table.path: /root + elastic.endpoint: fda-search-service:9200 zenodo: endpoint: https://sandbox.zenodo.org/ api_key: "${ZENODO_API_KEY}" \ No newline at end of file diff --git a/fda-table-service/rest-service/src/main/resources/application.yml b/fda-table-service/rest-service/src/main/resources/application.yml index ea38e7c2d131c0861582ac5aa1608ff123cbccab..e6e74dec02d72744c6dbe5064c85ea8227e307b4 100644 --- a/fda-table-service/rest-service/src/main/resources/application.yml +++ b/fda-table-service/rest-service/src/main/resources/application.yml @@ -32,6 +32,7 @@ eureka: fda: mapping.path: rest-service/src/main/resources table.path: rest-service/src/main/java/at/tuwien/userdb + elastic.endpoint: fda-search-service:9200 zenodo: endpoint: https://sandbox.zenodo.org/ api_key: "${ZENODO_API_KEY}" \ No newline at end of file diff --git a/fda-table-service/rest-service/src/test/resources/application.properties b/fda-table-service/rest-service/src/test/resources/application.properties index 5002d6fb8f86b0ef84d19bef2f1e169fbd910a59..8d1717d7e449e5a0b5056d43101938200a4e9dcb 100644 --- a/fda-table-service/rest-service/src/test/resources/application.properties +++ b/fda-table-service/rest-service/src/test/resources/application.properties @@ -1,3 +1,5 @@ +spring.profiles.active=test-noelastic + # disable discovery spring.cloud.discovery.enabled = false @@ -14,6 +16,9 @@ spring.jpa.database-platform=org.hibernate.dialect.H2Dialect spring.jpa.hibernate.ddl-auto=create-drop spring.jpa.show-sql=false +# disable elasticsearch +spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration,org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchRepositoriesAutoConfiguration,org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration + fda.mapping.path: /tmp fda.table.path: /tmp diff --git a/fda-table-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java b/fda-table-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..a510fc7b42add41f43c8d79fe8ca3334660025dc --- /dev/null +++ b/fda-table-service/services/src/main/java/at/tuwien/config/ElasticsearchConfig.java @@ -0,0 +1,35 @@ +package at.tuwien.config; + +import org.elasticsearch.client.RestHighLevelClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.elasticsearch.client.ClientConfiguration; +import org.springframework.data.elasticsearch.client.RestClients; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.ElasticsearchRestTemplate; +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; + +import java.util.Properties; + +@Configuration +public class ElasticsearchConfig { + + @Value("${fda.elastic.endpoint}") + private String elasticEndpoint; + + @Bean + public RestHighLevelClient client() { + ClientConfiguration clientConfiguration = ClientConfiguration.builder() + .connectedTo(elasticEndpoint) + .build(); + + return RestClients.create(clientConfiguration).rest(); + } + + @Bean + public ElasticsearchOperations elasticsearchTemplate() { + return new ElasticsearchRestTemplate(client()); + } +} \ No newline at end of file diff --git a/fda-table-service/services/src/main/java/at/tuwien/config/IndexInitializer.java b/fda-table-service/services/src/main/java/at/tuwien/config/IndexInitializer.java new file mode 100644 index 0000000000000000000000000000000000000000..ec0ab164070c9e64b8570de4d3e6426c4dff6748 --- /dev/null +++ b/fda-table-service/services/src/main/java/at/tuwien/config/IndexInitializer.java @@ -0,0 +1,39 @@ +package at.tuwien.config; + +import at.tuwien.entities.database.Database; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.env.Environment; +import org.springframework.core.env.Profiles; +import org.springframework.data.elasticsearch.core.ElasticsearchOperations; +import org.springframework.data.elasticsearch.core.mapping.IndexCoordinates; +import org.springframework.stereotype.Component; + +@Component +@Log4j2 +public class IndexInitializer { + + private final Environment environment; + private final ElasticsearchOperations elasticsearchOperations; + + @Autowired + public IndexInitializer(Environment environment, ElasticsearchOperations elasticsearchOperations) { + this.environment = environment; + this.elasticsearchOperations = elasticsearchOperations; + } + + @EventListener(ApplicationReadyEvent.class) + public void initIndex() { + if (environment.acceptsProfiles(Profiles.of("test-noelastic"))) { + return; + } + log.debug("creating index"); + IndexCoordinates indexCoordinates = IndexCoordinates.of("tblindex"); + if (!elasticsearchOperations.indexOps(indexCoordinates).exists()) { + elasticsearchOperations.indexOps(indexCoordinates).create(); + elasticsearchOperations.indexOps(indexCoordinates).createMapping(Database.class); + } + } +} diff --git a/fda-table-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseidxRepository.java b/fda-table-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseidxRepository.java new file mode 100644 index 0000000000000000000000000000000000000000..629d38c73ef5fa89b4ed3e1bc9ca9e5ba621980a --- /dev/null +++ b/fda-table-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseidxRepository.java @@ -0,0 +1,9 @@ +package at.tuwien.repository.elastic; + +import at.tuwien.entities.database.Database; +import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; +import org.springframework.stereotype.Repository; + +@Repository(value = "ElasticDatabaseRepository") +public interface DatabaseidxRepository extends ElasticsearchRepository<Database, Long> { +} diff --git a/fda-table-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseRepository.java b/fda-table-service/services/src/main/java/at/tuwien/repository/elastic/TableidxRepository.java similarity index 62% rename from fda-table-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseRepository.java rename to fda-table-service/services/src/main/java/at/tuwien/repository/elastic/TableidxRepository.java index aa60a6dbbcdd124de8a38c4098ee43c7115df16f..fb60eb355aaaf4e050471da84b444d91368aca38 100644 --- a/fda-table-service/services/src/main/java/at/tuwien/repository/elastic/DatabaseRepository.java +++ b/fda-table-service/services/src/main/java/at/tuwien/repository/elastic/TableidxRepository.java @@ -1,9 +1,9 @@ package at.tuwien.repository.elastic; -import at.tuwien.entities.database.Database; +import at.tuwien.entities.database.table.Table; import org.springframework.data.elasticsearch.repository.ElasticsearchRepository; import org.springframework.stereotype.Repository; @Repository(value = "ElasticDatabaseService") -public interface DatabaseRepository extends ElasticsearchRepository<Database, Long> { +public interface TableidxRepository extends ElasticsearchRepository<Table, Long> { } \ No newline at end of file