diff --git a/.gitignore b/.gitignore index 549e00a2a96fa9d7c5dbc9859664a78d980158c2..f93fd49ad4e76f8a804cea5748d6780ba1f71a58 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ target/ !**/src/main/**/target/ !**/src/test/**/target/ +# IDE +fda-analyse-service/venv + ### STS ### .apt_generated .classpath diff --git a/fda-container-service/rest-service/src/test/java/at/tuwien/service/ContainerServiceIntegrationTest.java b/fda-container-service/rest-service/src/test/java/at/tuwien/service/ContainerServiceIntegrationTest.java index c9e578456e40a39ecfe8e48b7119a7387589d153..47d5ad35cfa8c375fb14a65545b6190fdf8f0b8e 100644 --- a/fda-container-service/rest-service/src/test/java/at/tuwien/service/ContainerServiceIntegrationTest.java +++ b/fda-container-service/rest-service/src/test/java/at/tuwien/service/ContainerServiceIntegrationTest.java @@ -55,6 +55,7 @@ public class ContainerServiceIntegrationTest extends BaseUnitTest { @Transactional @BeforeEach public void beforeEach() { + afterEach(); /* create network */ dockerClient.createNetworkCmd() .withName("fda-userdb") diff --git a/fda-metadata-db/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java b/fda-metadata-db/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java index da9c5eb78246158b0a661efc0d9f5383a47af521..f37e071e389456e1b5c48adb926228fcad0fdace 100644 --- a/fda-metadata-db/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java +++ b/fda-metadata-db/entities/src/main/java/at/tuwien/entities/container/image/ContainerImage.java @@ -58,7 +58,7 @@ public class ContainerImage { private Integer defaultPort; @ToString.Include - @OneToMany(fetch = FetchType.EAGER, cascade = CascadeType.ALL) + @OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL) private List<ContainerImageEnvironmentItem> environment; @Column(nullable = false, updatable = false) diff --git a/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/Database.java b/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/Database.java index 7bef74053388be9d1aada44ad106de18682cab1f..69fcaa060b6d86c70182380b455e5d8eadcd3d2f 100644 --- a/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/Database.java +++ b/fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/Database.java @@ -35,7 +35,7 @@ public class Database { private Long id; @ToString.Include - @ManyToOne(fetch = FetchType.LAZY, cascade = CascadeType.ALL) + @ManyToOne(fetch = FetchType.EAGER, cascade = CascadeType.ALL) private Container container; @ToString.Include @@ -47,7 +47,7 @@ public class Database { private String internalName; @ToString.Exclude - @OneToMany(mappedBy = "id", fetch = FetchType.LAZY) + @OneToMany(mappedBy = "id", fetch = FetchType.EAGER, cascade = CascadeType.ALL) private List<Table> tables; @ToString.Include diff --git a/fda-table-service/pom.xml b/fda-table-service/pom.xml index 9837dee2b729b470307fe0ee8f01853da7e0df7e..1c7ae718b3d28b1b27e5882e1d6f4fdd405b2aeb 100644 --- a/fda-table-service/pom.xml +++ b/fda-table-service/pom.xml @@ -42,6 +42,10 @@ <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-web</artifactId> </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-validation</artifactId> + </dependency> <dependency> <groupId>org.springframework.cloud</groupId> <artifactId>spring-cloud-starter-bootstrap</artifactId> diff --git a/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java index 61c5509e7a424e772c46df8f953d51f1525d936e..9dce10338ef9ad0a83a53f6f5c39b57c24010fb6 100644 --- a/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java +++ b/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java @@ -20,6 +20,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; +import javax.validation.Valid; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; @@ -68,7 +69,7 @@ public class TableEndpoint { @ApiResponse(code = 422, message = "The ."), }) public ResponseEntity<TableBriefDto> create(@PathVariable("id") Long databaseId, - @RequestBody TableCreateDto createDto) + @Valid @RequestBody TableCreateDto createDto) throws ImageNotSupportedException, DatabaseConnectionException, TableMalformedException, DatabaseNotFoundException, DataProcessingException { final Table table = tableService.create(databaseId, createDto); @@ -102,7 +103,7 @@ public class TableEndpoint { @ApiResponse(code = 405, message = "The container is not running."), @ApiResponse(code = 409, message = "The container image is not supported."), }) - public ResponseEntity<TableDto> createViaCsv(@PathVariable("id") Long databaseId, @RequestBody TableCsvInformationDto tableCSVInformation) throws IOException { + public ResponseEntity<TableDto> createViaCsv(@PathVariable("id") Long databaseId, @Valid @RequestBody TableCsvInformationDto tableCSVInformation) throws IOException { final Table table = tableService.create(databaseId, tableCSVInformation); return ResponseEntity.status(HttpStatus.CREATED) .body(tableMapper.tableToTableDto(table)); diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java b/fda-table-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java index 8cc8006d8fd99e718b5997f28c1ce99823ca3476..f6693d63d4434644e9d78c2f862d92dd0f972e52 100644 --- a/fda-table-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java +++ b/fda-table-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java @@ -73,17 +73,6 @@ public class TableServiceUnitTest extends BaseUnitTest { }); } - @Test - public void findAll_noTable_fails() { - when(databaseRepository.findById(DATABASE_2_ID)) - .thenReturn(Optional.of(DATABASE_2)); - - /* test */ - assertThrows(TableNotFoundException.class, () -> { - tableService.findAll(DATABASE_2_ID); - }); - } - @Disabled("invalid mock") @Test public void delete_succeeds() throws TableNotFoundException, DatabaseConnectionException, TableMalformedException, diff --git a/fda-table-service/rest-service/src/test/resources/application.properties b/fda-table-service/rest-service/src/test/resources/application.properties index 2871cae9642987896d091460c42d1f27f857c25c..69df4a312304990e2f284318cd5428d960f14f0b 100644 --- a/fda-table-service/rest-service/src/test/resources/application.properties +++ b/fda-table-service/rest-service/src/test/resources/application.properties @@ -10,4 +10,5 @@ spring.datasource.url=jdbc:h2:mem:testdb spring.datasource.driverClassName=org.h2.Driver spring.datasource.username=sa spring.datasource.password=password -spring.jpa.database-platform=org.hibernate.dialect.H2Dialect \ No newline at end of file +spring.jpa.database-platform=org.hibernate.dialect.H2Dialect +spring.jpa.hibernate.ddl-auto=update \ No newline at end of file diff --git a/fda-table-service/services/src/main/java/at/tuwien/mapper/TableMapper.java b/fda-table-service/services/src/main/java/at/tuwien/mapper/TableMapper.java index 06f80f883bc932dc5f9bbc439104d81ef41af0b9..3341243d73ad471ed27f6c5fc1463210357ca0ad 100644 --- a/fda-table-service/services/src/main/java/at/tuwien/mapper/TableMapper.java +++ b/fda-table-service/services/src/main/java/at/tuwien/mapper/TableMapper.java @@ -21,8 +21,9 @@ public interface TableMapper { TableDto tableToTableDto(Table data); @Mappings({ - @Mapping(source = "columns", target = "columns", qualifiedByName = "columnMapping"), + @Mapping(target = "columns"), @Mapping(source = "name", target = "name"), + @Mapping(source = "name", target = "internalName", qualifiedByName = "columnSlug"), }) Table tableCreateDtoToTable(TableCreateDto data); diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java b/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java index 0141e0ff331b03992a81cc817d17f10e389853a7..47e6523e98633fd1073a6f574ad99341f6ddc201 100644 --- a/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java +++ b/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java @@ -117,7 +117,7 @@ public class PostgresService extends JdbcConnector { @Override public final PreparedStatement getCreateTableStatement(Connection connection, TableCreateDto createDto) throws DataProcessingException { - log.debug("create table columns {}", Arrays.toString(createDto.getColumns())); + log.debug("create table columns {}", Arrays.asList(createDto.getColumns())); final StringBuilder queryBuilder = new StringBuilder() .append("CREATE TABLE ") .append(tableMapper.columnNameToString(createDto.getName())) @@ -231,6 +231,7 @@ public class PostgresService extends JdbcConnector { */ private List<String> mockAnalyzeService(ColumnCreateDto[] columnDto) { final List<String> columns = new LinkedList<>(); + log.debug("will map column: {}", Arrays.asList(columnDto)); for (ColumnCreateDto column : columnDto) { final StringBuilder columnBuilder = new StringBuilder() .append(tableMapper.columnNameToString(column.getName())) diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java b/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java index 252bb5b71029bc0e3590410a24cea95c44ec68c8..2997fe15c95d2d870abb1403c1d4739f86f296e8 100644 --- a/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java +++ b/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java @@ -53,6 +53,7 @@ public class TableService { this.tableMapper = tableMapper; } + @Transactional public List<Table> findAll(Long databaseId) throws DatabaseNotFoundException, TableNotFoundException { final Optional<Database> database; try { @@ -65,19 +66,17 @@ public class TableService { log.error("Unable to find database {}", databaseId); throw new DatabaseNotFoundException("Unable to find database."); } - if (database.get().getTables().size() == 0) { - log.error("Unable to find tables for database {}.", database); - throw new TableNotFoundException("Unable to find tables."); - } return database.get().getTables(); } + @Transactional public void delete(Long databaseId, Long tableId) throws TableNotFoundException, DatabaseConnectionException, TableMalformedException, DataProcessingException { final Table table = findById(databaseId, tableId); postgresService.deleteTable(table); tableRepository.deleteById(tableId); } + @Transactional public Table findById(Long databaseId, Long tableId) throws TableNotFoundException { final Database database = Database.builder() .id(databaseId) @@ -96,7 +95,6 @@ public class TableService { log.error("no database with this id found in metadata database"); throw new DatabaseNotFoundException("database not found in metadata database"); } - log.debug("retrieved db {}", database); if (!database.get().getContainer().getImage().getRepository().equals("postgres")) { log.error("Right now only PostgreSQL is supported!"); throw new ImageNotSupportedException("Currently only PostgreSQL is supported"); @@ -112,15 +110,32 @@ public class TableService { /* save in metadata db */ postgresService.createTable(database, createDto); - final Table table = tableMapper.tableCreateDtoToTable(createDto); - table.setDatabase(database); - table.setInternalName(tableMapper.columnNameToString(table.getName())); - final Table out = tableRepository.save(table); - log.debug("saved table {}", out); + final Table mappedTable = tableMapper.tableCreateDtoToTable(createDto); + mappedTable.setDatabase(database); + mappedTable.setTdbid(databaseId); + mappedTable.setColumns(List.of()); + final Table table; + try { + table = tableRepository.save(mappedTable); + } catch (EntityNotFoundException e) { + throw new DataProcessingException("failed to create table compound key", e); + } + table.getColumns().forEach(column -> { + column.setCdbid(databaseId); + column.setTid(table.getId()); + }); + final Table out; + try { + out = tableRepository.save(table); + } catch (EntityNotFoundException e) { + throw new DataProcessingException("failed to create column compound key", e); + } + log.debug("saved table: {}", out); log.info("Created table {} in database {}", out.getName(), out.getDatabase().getId()); return out; } + @Transactional public QueryResultDto insert(Long databaseId, Long tableId, MultipartFile file) throws TableNotFoundException, ImageNotSupportedException, DatabaseNotFoundException, DatabaseConnectionException, DataProcessingException { Table t = findById(databaseId, tableId); @@ -192,10 +207,10 @@ public class TableService { String[] header = mapReader.getHeader(true); return header; - } catch(IOException e) { + } catch (IOException e) { e.printStackTrace(); } finally { - if( mapReader != null ) { + if (mapReader != null) { mapReader.close(); } } @@ -206,7 +221,7 @@ public class TableService { public QueryResultDto showData(Long databaseId, Long tableId) throws ImageNotSupportedException, DatabaseNotFoundException, TableNotFoundException, DatabaseConnectionException, DataProcessingException { QueryResultDto queryResult = postgresService.getAllRows(findDatabase(databaseId), findById(databaseId, tableId)); - for (Map<String, Object> m : queryResult.getResult() ) { + for (Map<String, Object> m : queryResult.getResult()) { for (Map.Entry<String, Object> entry : m.entrySet()) { log.debug("{}: {}", entry.getKey(), entry.getValue()); } @@ -229,7 +244,7 @@ public class TableService { c.setType(tableCSVInformation.getColumns().get(i)); c.setNullAllowed(true); //TODO FIX THAT not only id is primary key - if(header[i].equals("id")) { + if (header[i].equals("id")) { c.setPrimaryKey(true); } else { c.setPrimaryKey(false); @@ -258,7 +273,7 @@ public class TableService { MultipartFile multipartFile = new MockMultipartFile(tableCSVInformation.getFileLocation(), tableCSVInformation.getFileLocation(), contentType, content); Files.deleteIfExists(path); - return create(databaseId, multipartFile,tableCSVInformation); + return create(databaseId, multipartFile, tableCSVInformation); } diff --git a/fda-ui/package-lock.json b/fda-ui/package-lock.json index b297a203c30d1c0526180bf2c0851c60851a238b..4e2ba32502ee636a9012de8cddedd42e6d672c13 100644 --- a/fda-ui/package-lock.json +++ b/fda-ui/package-lock.json @@ -2239,6 +2239,11 @@ "picomatch": "^2.0.4" } }, + "append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY=" + }, "aproba": { "version": "1.2.0", "resolved": "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz", @@ -2780,6 +2785,38 @@ "resolved": "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=" }, + "busboy": { + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", + "integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=", + "requires": { + "dicer": "0.2.5", + "readable-stream": "1.1.x" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", @@ -3732,6 +3769,38 @@ "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", "dev": true }, + "dicer": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", + "integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=", + "requires": { + "readable-stream": "1.1.x", + "streamsearch": "0.1.2" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "diffie-hellman": { "version": "5.0.3", "resolved": "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz", @@ -6425,6 +6494,21 @@ "resolved": "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "multer": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.2.tgz", + "integrity": "sha512-xY8pX7V+ybyUpbYMxtjM9KAiD9ixtg5/JkeKUTD6xilfDv0vzzOFcCp4Ljb1UU3tSOM3VTZtKo63OmzOrGi3Cg==", + "requires": { + "append-field": "^1.0.0", + "busboy": "^0.2.11", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.1", + "object-assign": "^4.1.1", + "on-finished": "^2.3.0", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + } + }, "multimap": { "version": "1.1.0", "resolved": "https://registry.yarnpkg.com/multimap/-/multimap-1.1.0.tgz", @@ -8977,6 +9061,11 @@ "resolved": "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz", "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" }, + "streamsearch": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", + "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=" + }, "strict-uri-encode": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz",