diff --git a/docker-compose.yml b/docker-compose.yml index c19bc103d58338f7f917ea095b2d31c9aab36942..4c9736d360bcefca6000b7acebb5f04a7e0d03cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -160,6 +160,7 @@ services: ports: - 9094:9094 volumes: + - /var/run/docker.sock:/var/run/docker.sock - /tmp:/tmp links: - fda-metadata-db diff --git a/fda-table-service/pom.xml b/fda-table-service/pom.xml index 25d843b452014e1c7f6319faa8541a5dd3f318ff..9837dee2b729b470307fe0ee8f01853da7e0df7e 100644 --- a/fda-table-service/pom.xml +++ b/fda-table-service/pom.xml @@ -181,7 +181,9 @@ <exclude>at/tuwien/mapper/**/*</exclude> <exclude>at/tuwien/exception/**/*</exclude> <exclude>at/tuwien/utils/**/*</exclude> + <exclude>at/tuwien/config/**/*</exclude> <exclude>**/FdaTableServiceApplication.class</exclude> + <exclude>**/JdbcConnector.class</exclude> </excludes> </configuration> <executions> diff --git a/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java index cd9bed3897dc9e6d6bff3730f2695f4fa84053df..e9ac17495553e415d209d3955cd3969abbd847cc 100644 --- a/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java +++ b/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java @@ -55,7 +55,7 @@ public class TableEndpoint { } @PostMapping("/table") - @ApiOperation(value = "Create a table", notes = "Creates a new table for a database, requires a running container. For the colum definition use the following example: [{\"name\": \"Ticker Symbol\", \"primaryKey\": true, \"type\": \"STRING\", \"nullAllowed\": false, \"checkExpression\": null, \"foreignKey\": null},{\"name\": \"Accounts Payable\", \"primaryKey\": false, \"type\": \"NUMBER\", \"nullAllowed\": false, \"checkExpression\": \"Accounts Payable > 0\", \"foreignKey\": null},{\"name\": \"Company\", \"primaryKey\": false, \"type\": \"STRING\", \"nullAllowed\": false, \"checkExpression\": null, \"foreignKey\": null}]") + @ApiOperation(value = "Create a table", notes = "Creates a new table for a database, requires a running container.") @ApiResponses({ @ApiResponse(code = 201, message = "The table was created."), @ApiResponse(code = 400, message = "The creation form contains invalid data."), @@ -65,9 +65,10 @@ public class TableEndpoint { @ApiResponse(code = 409, message = "The container image is not supported."), @ApiResponse(code = 422, message = "The ."), }) - public ResponseEntity<TableBriefDto> create(@PathVariable("id") Long databaseId, @RequestBody TableCreateDto createDto) + public ResponseEntity<TableBriefDto> create(@PathVariable("id") Long databaseId, + @RequestBody TableCreateDto createDto) throws ImageNotSupportedException, DatabaseConnectionException, TableMalformedException, - DatabaseNotFoundException { + DatabaseNotFoundException, DataProcessingException { final Table table = tableService.create(databaseId, createDto); return ResponseEntity.status(HttpStatus.CREATED) .body(tableMapper.tableToTableBriefDto(table)); @@ -80,7 +81,8 @@ public class TableEndpoint { @ApiResponse(code = 401, message = "Not authorized to list all tables."), @ApiResponse(code = 404, message = "Table not found in metadata database."), }) - public ResponseEntity<TableDto> findById(@PathVariable("id") Long databaseId, @PathVariable("tableId") Long tableId) throws TableNotFoundException { + public ResponseEntity<TableDto> findById(@PathVariable("id") Long databaseId, @PathVariable("tableId") Long tableId) + throws TableNotFoundException { final Table table = tableService.findById(databaseId, tableId); return ResponseEntity.ok(tableMapper.tableToTableDto(table)); } @@ -93,7 +95,8 @@ public class TableEndpoint { @ApiResponse(code = 401, message = "Not authorized to update tables."), @ApiResponse(code = 404, message = "The table is not found in database."), }) - public ResponseEntity<TableBriefDto> update(@PathVariable("id") Long databaseId, @PathVariable("tableId") Long tableId) { + public ResponseEntity<TableBriefDto> update(@PathVariable("id") Long databaseId, + @PathVariable("tableId") Long tableId) { // TODO return ResponseEntity.unprocessableEntity().body(new TableBriefDto()); } @@ -106,7 +109,9 @@ public class TableEndpoint { @ApiResponse(code = 404, message = "The table is not found in database."), }) @ResponseStatus(HttpStatus.OK) - public void delete(@PathVariable("id") Long databaseId, @PathVariable("tableId") Long tableId) throws TableNotFoundException, DatabaseConnectionException, TableMalformedException { + public void delete(@PathVariable("id") Long databaseId, @PathVariable("tableId") Long tableId) + throws TableNotFoundException, DatabaseConnectionException, TableMalformedException, + DataProcessingException { tableService.delete(databaseId, tableId); } @@ -120,7 +125,9 @@ public class TableEndpoint { @ApiResponse(code = 422, message = "The csv was not processible."), }) /* FIXME: this should be in a different endpoint */ - public ResponseEntity<QueryResultDto> insert(@PathVariable("id") Long databaseId, @PathVariable("tableId") Long tableId, @RequestParam("file") MultipartFile file) throws Exception { + public ResponseEntity<QueryResultDto> insert(@PathVariable("id") Long databaseId, + @PathVariable("tableId") Long tableId, + @RequestParam("file") MultipartFile file) throws Exception { final QueryResultDto queryResult = tableService.insert(databaseId, tableId, file); return ResponseEntity.ok(queryResultMapper.queryResultToQueryResultDto(queryResult)); } @@ -132,8 +139,10 @@ public class TableEndpoint { @ApiResponse(code = 401, message = "Not authorized to list all tables."), }) /* FIXME: this should be a different endpoint */ - public ResponseEntity<QueryResultDto> showData(@PathVariable("id") Long databaseId, @PathVariable("tableId") Long tableId) - throws DatabaseNotFoundException, ImageNotSupportedException, TableNotFoundException, DatabaseConnectionException { + public ResponseEntity<QueryResultDto> showData(@PathVariable("id") Long databaseId, + @PathVariable("tableId") Long tableId) + throws DatabaseNotFoundException, ImageNotSupportedException, TableNotFoundException, + DatabaseConnectionException, DataProcessingException { final QueryResultDto queryResult = tableService.showData(databaseId, tableId); return ResponseEntity.ok(queryResultMapper.queryResultToQueryResultDto(queryResult)); } diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/BaseUnitTest.java b/fda-table-service/rest-service/src/test/java/at/tuwien/BaseUnitTest.java index b8c3fd83567ba2ecb1ebe90ef21ab922fce6a6ce..ac7327bca19963b7a854775efcdcf64d5e68f1a1 100644 --- a/fda-table-service/rest-service/src/test/java/at/tuwien/BaseUnitTest.java +++ b/fda-table-service/rest-service/src/test/java/at/tuwien/BaseUnitTest.java @@ -7,6 +7,7 @@ import at.tuwien.entities.container.image.ContainerImage; import at.tuwien.entities.container.image.ContainerImageEnvironmentItem; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.table.Table; +import at.tuwien.entities.database.table.columns.TableColumn; import org.springframework.test.context.TestPropertySource; import java.time.Instant; @@ -22,6 +23,11 @@ public abstract class BaseUnitTest { public final static String DATABASE_1_INTERNALNAME = "nyse"; public final static String TABLE_1_DESCRIPTION = "New York Stock Exchange"; + public final static Long DATABASE_2_ID = 2L; + public final static String DATABASE_2_NAME = "RIVER"; + public final static String DATABASE_2_INTERNALNAME = "river"; + public final static String TABLE_2_DESCRIPTION = "River Data"; + public final static Long TABLE_1_ID = 1L; public final static String TABLE_1_NAME = "Stock Exchange"; public final static String TABLE_1_INTERNALNAME = "stock_exchange"; @@ -29,26 +35,35 @@ public abstract class BaseUnitTest { public final static Long COLUMN_1_ID = 1L; public final static Boolean COLUMN_1_PRIMARY = true; public final static String COLUMN_1_NAME = "Min"; + public final static String COLUMN_1_INTERNAL_NAME = "min"; public final static ColumnTypeDto COLUMN_1_TYPE = ColumnTypeDto.NUMBER; public final static Boolean COLUMN_1_NULL = false; + public final static Long COLUMN_2_ID = 2L; public final static Boolean COLUMN_2_PRIMARY = false; public final static String COLUMN_2_NAME = "Max"; + public final static String COLUMN_2_INTERNAL_NAME = "max"; public final static ColumnTypeDto COLUMN_2_TYPE = ColumnTypeDto.NUMBER; public final static Boolean COLUMN_2_NULL = true; + public final static Long COLUMN_3_ID = 3L; public final static Boolean COLUMN_3_PRIMARY = false; public final static String COLUMN_3_NAME = "Buy"; + public final static String COLUMN_3_INTERNAL_NAME = "buy"; public final static ColumnTypeDto COLUMN_3_TYPE = ColumnTypeDto.NUMBER; public final static Boolean COLUMN_3_NULL = true; + public final static Long COLUMN_4_ID = 4L; public final static Boolean COLUMN_4_PRIMARY = false; public final static String COLUMN_4_NAME = "Sell"; + public final static String COLUMN_4_INTERNAL_NAME = "sell"; public final static ColumnTypeDto COLUMN_4_TYPE = ColumnTypeDto.NUMBER; public final static Boolean COLUMN_4_NULL = true; + public final static Long COLUMN_5_ID = 5L; public final static Boolean COLUMN_5_PRIMARY = false; public final static String COLUMN_5_NAME = "Description"; + public final static String COLUMN_5_INTERNAL_NAME = "description"; public final static ColumnTypeDto COLUMN_5_TYPE = ColumnTypeDto.TEXT; public final static Boolean COLUMN_5_NULL = true; @@ -69,6 +84,9 @@ public abstract class BaseUnitTest { .value("postgres") .build()); + public final static List<String> IMAGE_1_ENVIRONMENT = List.of("POSTGRES_USER=postgres", + "POSTGRES_PASSWORD=postgres", "POSTGRES_DB=" + DATABASE_1_INTERNALNAME); + public final static ContainerImage IMAGE_1 = ContainerImage.builder() .id(IMAGE_1_ID) .repository(IMAGE_1_REPOSITORY) @@ -81,14 +99,18 @@ public abstract class BaseUnitTest { .build(); public final static Long CONTAINER_1_ID = 1L; - public final static String CONTAINER_1_HASH = "deadbeef"; + public static String CONTAINER_1_HASH = "deadbeef"; public final static ContainerImage CONTAINER_1_IMAGE = IMAGE_1; public final static String CONTAINER_1_NAME = "u01"; public final static String CONTAINER_1_INTERNALNAME = "localhost"; - public final static String CONTAINER_1_DATABASE = "univie"; - public final static String CONTAINER_1_IP = "231.145.98.83"; public final static Instant CONTAINER_1_CREATED = Instant.now().minus(1, HOURS); + public final static Long CONTAINER_2_ID = 2L; + public final static String CONTAINER_2_HASH = "deadbeef"; + public final static String CONTAINER_2_NAME = "u02"; + public final static String CONTAINER_2_INTERNALNAME = "not3x1st1ng"; + public final static Instant CONTAINER_2_CREATED = Instant.now().minus(1, HOURS); + public final static Container CONTAINER_1 = Container.builder() .id(CONTAINER_1_ID) .name(CONTAINER_1_NAME) @@ -98,12 +120,73 @@ public abstract class BaseUnitTest { .containerCreated(CONTAINER_1_CREATED) .build(); + public final static Container CONTAINER_2 = Container.builder() + .id(CONTAINER_2_ID) + .name(CONTAINER_2_NAME) + .internalName(CONTAINER_2_INTERNALNAME) + .image(CONTAINER_1_IMAGE) + .hash(CONTAINER_2_HASH) + .containerCreated(CONTAINER_2_CREATED) + .build(); + + public final static List<TableColumn> TABLE_1_COLUMNS = List.of(TableColumn.builder() + .id(COLUMN_1_ID) + .cdbid(DATABASE_1_ID) + .tid(TABLE_1_ID) + .name(COLUMN_1_NAME) + .internalName(COLUMN_1_INTERNAL_NAME) + .columnType(COLUMN_1_TYPE.name()) + .isNullAllowed(COLUMN_1_NULL) + .isPrimaryKey(COLUMN_1_PRIMARY) + .build(), + TableColumn.builder() + .id(COLUMN_2_ID) + .cdbid(DATABASE_1_ID) + .tid(TABLE_1_ID) + .name(COLUMN_2_NAME) + .internalName(COLUMN_2_INTERNAL_NAME) + .columnType(COLUMN_2_TYPE.name()) + .isNullAllowed(COLUMN_2_NULL) + .isPrimaryKey(COLUMN_2_PRIMARY) + .build(), + TableColumn.builder() + .id(COLUMN_3_ID) + .cdbid(DATABASE_1_ID) + .tid(TABLE_1_ID) + .name(COLUMN_3_NAME) + .internalName(COLUMN_3_INTERNAL_NAME) + .columnType(COLUMN_3_TYPE.name()) + .isNullAllowed(COLUMN_3_NULL) + .isPrimaryKey(COLUMN_3_PRIMARY) + .build(), + TableColumn.builder() + .id(COLUMN_4_ID) + .cdbid(DATABASE_1_ID) + .tid(TABLE_1_ID) + .name(COLUMN_4_NAME) + .internalName(COLUMN_4_INTERNAL_NAME) + .columnType(COLUMN_4_TYPE.name()) + .isNullAllowed(COLUMN_4_NULL) + .isPrimaryKey(COLUMN_4_PRIMARY) + .build(), + TableColumn.builder() + .id(COLUMN_5_ID) + .cdbid(DATABASE_1_ID) + .tid(TABLE_1_ID) + .name(COLUMN_5_NAME) + .internalName(COLUMN_5_INTERNAL_NAME) + .columnType(COLUMN_5_TYPE.name()) + .isNullAllowed(COLUMN_5_NULL) + .isPrimaryKey(COLUMN_5_PRIMARY) + .build()); + public final static Table TABLE_1 = Table.builder() .id(TABLE_1_ID) .created(Instant.now()) .internalName(TABLE_1_INTERNALNAME) .name(TABLE_1_NAME) .lastModified(Instant.now()) + .columns(TABLE_1_COLUMNS) .tdbid(DATABASE_1_ID) .build(); @@ -117,6 +200,17 @@ public abstract class BaseUnitTest { .internalName(DATABASE_1_INTERNALNAME) .build(); + /* no connection */ + public final static Database DATABASE_2 = Database.builder() + .id(DATABASE_2_ID) + .created(Instant.now().minus(1, HOURS)) + .lastModified(Instant.now()) + .isPublic(false) + .name(DATABASE_2_NAME) + .container(CONTAINER_2) + .internalName(DATABASE_2_INTERNALNAME) + .build(); + public final static ColumnCreateDto[] COLUMNS5 = new ColumnCreateDto[]{ ColumnCreateDto.builder() .type(COLUMN_1_TYPE) @@ -149,4 +243,6 @@ public abstract class BaseUnitTest { .primaryKey(COLUMN_5_PRIMARY) .build()}; + public final static String COLUMNS5_CREATE = "CREATE TABLE " + TABLE_1_INTERNALNAME + "(Min INTEGER NOT NULL PRIMARY KEY,Max INTEGER NULL,Buy INTEGER NULL,Sell INTEGER NULL,Description TEXT NULL);"; + } diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java b/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java index cc817a7ef1e81b0c281ac8f173c4de12ad68c10f..0ca5b7b01e1334a62482237bf75d9ad13cd95efa 100644 --- a/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java +++ b/fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java @@ -60,7 +60,7 @@ public class TableEndpointUnitTest extends BaseUnitTest { @Test public void create_succeeds() throws DatabaseConnectionException, TableMalformedException, - DatabaseNotFoundException, ImageNotSupportedException, TableNotFoundException { + DatabaseNotFoundException, ImageNotSupportedException, TableNotFoundException, DataProcessingException { final TableCreateDto request = TableCreateDto.builder() .name(TABLE_1_NAME) .description(TABLE_1_DESCRIPTION) @@ -78,7 +78,7 @@ public class TableEndpointUnitTest extends BaseUnitTest { @Test public void create_databaseNotFound_fails() throws DatabaseConnectionException, TableMalformedException, - DatabaseNotFoundException, ImageNotSupportedException { + DatabaseNotFoundException, ImageNotSupportedException, DataProcessingException { final TableCreateDto request = TableCreateDto.builder() .name(TABLE_1_NAME) .description(TABLE_1_DESCRIPTION) @@ -97,7 +97,7 @@ public class TableEndpointUnitTest extends BaseUnitTest { @Test public void create_tableNotFound_fails() throws DatabaseConnectionException, TableMalformedException, - DatabaseNotFoundException, ImageNotSupportedException { + DatabaseNotFoundException, ImageNotSupportedException, DataProcessingException { final TableCreateDto request = TableCreateDto.builder() .name(TABLE_1_NAME) .description(TABLE_1_DESCRIPTION) @@ -183,7 +183,8 @@ public class TableEndpointUnitTest extends BaseUnitTest { } @Test - public void delete_notFound_fails() throws TableNotFoundException, DatabaseConnectionException, TableMalformedException { + public void delete_notFound_fails() throws TableNotFoundException, DatabaseConnectionException, + TableMalformedException, DataProcessingException { doThrow(TableNotFoundException.class) .when(tableService) .delete(DATABASE_1_ID, TABLE_1_ID); @@ -195,7 +196,8 @@ public class TableEndpointUnitTest extends BaseUnitTest { } @Test - public void delete_succeeds() throws TableNotFoundException, DatabaseConnectionException, TableMalformedException { + public void delete_succeeds() throws TableNotFoundException, DatabaseConnectionException, TableMalformedException, + DataProcessingException { /* test */ tableEndpoint.delete(DATABASE_1_ID, TABLE_1_ID); } diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/service/PostgresIntegrationTest.java b/fda-table-service/rest-service/src/test/java/at/tuwien/service/PostgresIntegrationTest.java new file mode 100644 index 0000000000000000000000000000000000000000..1093a4b4753a88bbcdfe6057a29fdf97cb82d50f --- /dev/null +++ b/fda-table-service/rest-service/src/test/java/at/tuwien/service/PostgresIntegrationTest.java @@ -0,0 +1,186 @@ +package at.tuwien.service; + +import at.tuwien.BaseUnitTest; +import at.tuwien.api.database.table.TableCreateDto; +import at.tuwien.exception.DataProcessingException; +import at.tuwien.exception.DatabaseConnectionException; +import at.tuwien.exception.TableMalformedException; +import at.tuwien.exception.TableNotFoundException; +import at.tuwien.mapper.TableMapper; +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.CreateContainerResponse; +import com.github.dockerjava.api.exception.NotModifiedException; +import com.github.dockerjava.api.model.HostConfig; +import com.github.dockerjava.api.model.PortBinding; +import com.netflix.discovery.converters.Auto; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; + +@SpringBootTest +@ExtendWith(SpringExtension.class) +public class PostgresIntegrationTest extends BaseUnitTest { + + @Autowired + private PostgresService postgresService; + + @Autowired + private TableMapper tableMapper; + + @Autowired + private Properties postgresProperties; + + @Autowired + private HostConfig hostConfig; + + @Autowired + private DockerClient dockerClient; + + @BeforeEach + public void beforeEach() throws InterruptedException { + afterEach(); + /* create container */ + final CreateContainerResponse request = dockerClient.createContainerCmd("postgres:latest") + .withEnv(IMAGE_1_ENVIRONMENT) + .withHostConfig(hostConfig.withNetworkMode("bridge") + .withPortBindings(PortBinding.parse("5433:5432"))) + .withName(CONTAINER_1_INTERNALNAME) + .withHostName(CONTAINER_1_INTERNALNAME) + .exec(); + System.out.println("CREATE CONTAINER " + CONTAINER_1_INTERNALNAME); + /* start container */ + dockerClient.startContainerCmd(request.getId()).exec(); + System.out.println("START CONTAINER " + CONTAINER_1_INTERNALNAME); + CONTAINER_1_HASH = request.getId(); + CONTAINER_1.setHash(CONTAINER_1_HASH); + System.out.println("Wait 5s for DB to get up"); + Thread.sleep(5 * 1000); + } + + @AfterEach + public void afterEach() { + /* stop containers and remove them */ + dockerClient.listContainersCmd() + .withShowAll(true) + .exec() + .forEach(container -> { + try { + dockerClient.stopContainerCmd(container.getId()).exec(); + System.out.println("STOP CONTAINER " + Arrays.toString(container.getNames())); + } catch (NotModifiedException e) { + // ignore + } + dockerClient.removeContainerCmd(container.getId()).exec(); + System.out.println("DELETE CONTAINER " + Arrays.toString(container.getNames())); + }); + } + + @Test + public void createTable_succeeds() throws DatabaseConnectionException, TableMalformedException, DataProcessingException { + final TableCreateDto request = TableCreateDto.builder() + .name(TABLE_1_NAME) + .description(TABLE_1_NAME) + .columns(COLUMNS5) + .build(); + + /* test */ + postgresService.createTable(DATABASE_1, request); + } + + @Test + public void createTable_noConnection_fails() { + final TableCreateDto request = TableCreateDto.builder() + .name(TABLE_1_NAME) + .description(TABLE_1_NAME) + .columns(COLUMNS5) + .build(); + + /* test */ + assertThrows(DatabaseConnectionException.class, () -> { + postgresService.createTable(DATABASE_2, request); + }); + } + + @Disabled("cannot test") + @Test + public void createTable_noSql_fails() throws DataProcessingException { + final TableCreateDto request = TableCreateDto.builder() + .name(TABLE_1_NAME) + .description(TABLE_1_NAME) + .columns(COLUMNS5) + .build(); + final PostgresService mockService = mock(PostgresService.class); + + /* test */ + assertThrows(TableMalformedException.class, () -> { + postgresService.createTable(DATABASE_1, request); + }); + } + + @Disabled("not testable for me") + @Test + public void insertIntoTable_succeeds() throws SQLException, DatabaseConnectionException, DataProcessingException { + final Connection connection = DriverManager.getConnection("jdbc:postgresql://" + CONTAINER_1_INTERNALNAME + ":" + IMAGE_1_PORT + "/" + DATABASE_1_INTERNALNAME, postgresProperties); + connection.prepareStatement(COLUMNS5_CREATE).execute(); + final List<Map<String, Object>> data = List.of(Map.of(COLUMN_1_NAME, 1, COLUMN_2_NAME, 2, COLUMN_3_NAME, 3, COLUMN_4_NAME, 4, COLUMN_5_NAME, "Description")); + final List<String> headers = List.of(COLUMN_1_NAME, COLUMN_2_NAME, COLUMN_3_NAME, COLUMN_4_NAME, COLUMN_5_NAME); + + postgresService.insertIntoTable(DATABASE_1, TABLE_1, data, headers); + } + + @Disabled("not testable for me") + @Test + public void insertIntoTable_noConnection_fails() { + + } + + @Disabled("not testable for me") + @Test + public void insertIntoTable_noSql_fails() { + + } + + @Disabled("not testable for me") + @Test + public void getAllRows_succeeds() { + + } + + @Disabled("not testable for me") + @Test + public void getAllRows_noSql_fails() { + + } + + @Disabled("not testable for me") + @Test + public void getCreateTableStatement_noSql_fails() { + + } + + @Disabled("not testable for me") + @Test + public void insertStatement_succeeds() { + + } + + @Disabled("not testable for me") + @Test + public void getDeleteStatement_noSql_fails() { + + } + +} diff --git a/fda-table-service/rest-service/src/test/java/at/tuwien/service/PostgresUnitTest.java b/fda-table-service/rest-service/src/test/java/at/tuwien/service/PostgresUnitTest.java deleted file mode 100644 index 6d1adb5a841fa094b50ac08422a97aa660ea4b04..0000000000000000000000000000000000000000 --- a/fda-table-service/rest-service/src/test/java/at/tuwien/service/PostgresUnitTest.java +++ /dev/null @@ -1,37 +0,0 @@ -package at.tuwien.service; - -import at.tuwien.BaseUnitTest; -import at.tuwien.api.database.table.TableBriefDto; -import at.tuwien.api.database.table.TableCreateDto; -import at.tuwien.api.database.table.TableDto; -import at.tuwien.endpoints.TableEndpoint; -import at.tuwien.entities.database.Database; -import at.tuwien.exception.*; -import at.tuwien.repository.DatabaseRepository; -import at.tuwien.repository.TableRepository; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.sql.SQLException; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.when; - -@SpringBootTest -@ExtendWith(SpringExtension.class) -public class PostgresUnitTest extends BaseUnitTest { - - -} diff --git a/fda-table-service/services/src/main/java/at/tuwien/config/DockerConfig.java b/fda-table-service/services/src/main/java/at/tuwien/config/DockerConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..25021a8dad22ec6bb27acf72cbbad9570c946472 --- /dev/null +++ b/fda-table-service/services/src/main/java/at/tuwien/config/DockerConfig.java @@ -0,0 +1,37 @@ +package at.tuwien.config; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.model.HostConfig; +import com.github.dockerjava.api.model.RestartPolicy; +import com.github.dockerjava.core.DefaultDockerClientConfig; +import com.github.dockerjava.core.DockerClientBuilder; +import com.github.dockerjava.core.DockerClientConfig; +import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; +import com.github.dockerjava.transport.DockerHttpClient; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class DockerConfig { + + @Bean + public HostConfig hostConfig() { + return HostConfig.newHostConfig() + .withRestartPolicy(RestartPolicy.alwaysRestart()); + } + + @Bean + public DockerClient dockerClientConfiguration() { + final DockerClientConfig dockerClientConfig = DefaultDockerClientConfig.createDefaultConfigBuilder() + .withDockerHost("unix:///var/run/docker.sock") + .build(); + final DockerHttpClient dockerHttpClient = new ApacheDockerHttpClient.Builder() + .dockerHost(dockerClientConfig.getDockerHost()) + .sslConfig(dockerClientConfig.getSSLConfig()) + .build(); + return DockerClientBuilder.getInstance() + .withDockerHttpClient(dockerHttpClient) + .build(); + } + +} diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/JdbcConnector.java b/fda-table-service/services/src/main/java/at/tuwien/service/JdbcConnector.java index 03aae55ab3a599fda8dc6d9e2bf9c543a18b5d63..cd8f5372d6d9f7653c59af7c5129838715780903 100644 --- a/fda-table-service/services/src/main/java/at/tuwien/service/JdbcConnector.java +++ b/fda-table-service/services/src/main/java/at/tuwien/service/JdbcConnector.java @@ -3,6 +3,7 @@ package at.tuwien.service; import at.tuwien.api.database.table.TableCreateDto; import at.tuwien.entities.database.table.Table; +import at.tuwien.exception.DataProcessingException; import java.sql.*; import java.util.List; @@ -15,9 +16,9 @@ public abstract class JdbcConnector { return DriverManager.getConnection(url, properties); } - abstract PreparedStatement getCreateTableStatement(Connection connection, TableCreateDto createDto) throws SQLException; + abstract PreparedStatement getCreateTableStatement(Connection connection, TableCreateDto createDto) throws DataProcessingException; abstract String insertStatement(List<Map<String, Object>> processedData, Table t, List<String> headers); - abstract PreparedStatement getDeleteStatement(Connection connection, Table table) throws SQLException; + abstract PreparedStatement getDeleteStatement(Connection connection, Table table) throws DataProcessingException; } diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java b/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java index cf3c175fb511c3bc94e662d1d7888ef1e16f71ef..c1a420ebaeb278825dd60fbea7688ab12b19f157 100644 --- a/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java +++ b/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java @@ -48,13 +48,13 @@ public class PostgresService extends JdbcConnector { try { connection = open(URL, postgresProperties); } catch (SQLException e) { - log.error("Could not connect to the database container, is it running from Docker container? IT DOES NOT WORK FROM IDE! URL: {} Params: {}", URL, postgresProperties); + log.error("Could not connect to the database container, is it running from Docker container? URL: {} Params: {}", URL, postgresProperties); throw new DatabaseConnectionException("Could not connect to the database container, is it running at: " + URL, e); } return connection; } - public void createTable(Database database, TableCreateDto createDto) throws DatabaseConnectionException, TableMalformedException { + public void createTable(Database database, TableCreateDto createDto) throws DatabaseConnectionException, TableMalformedException, DataProcessingException { try { final PreparedStatement statement = getCreateTableStatement(getConnection(database), createDto); statement.execute(); @@ -73,7 +73,7 @@ public class PostgresService extends JdbcConnector { } catch (DatabaseConnectionException e) { log.error("Problem with connecting to the database while selecting from query store: {}", e.getMessage()); throw new DatabaseConnectionException("database connection problem with query store", e); - } catch (SQLException e) { + } catch (SQLException | NullPointerException e) { log.error("The SQL statement seems to contain invalid syntax: {}", e.getMessage()); throw new DataProcessingException("invalid syntax", e); } @@ -86,7 +86,7 @@ public class PostgresService extends JdbcConnector { * @param t * @return */ - public QueryResultDto getAllRows(Database database, Table t) throws DatabaseConnectionException { + public QueryResultDto getAllRows(Database database, Table t) throws DatabaseConnectionException, DataProcessingException { try { Connection connection = getConnection(database); PreparedStatement statement = connection.prepareStatement(selectStatement(t)); @@ -103,14 +103,13 @@ public class PostgresService extends JdbcConnector { qr.setResult(res); return qr; } catch (SQLException e) { - log.debug(e.getMessage()); - log.error("The SQL statement seems to contain invalid syntax"); + log.error("The SQL statement seems to contain invalid syntax: {}", e.getMessage()); + throw new DataProcessingException("invalid syntax", e); } - return null; } @Override - public final PreparedStatement getCreateTableStatement(Connection connection, TableCreateDto createDto) throws SQLException { + public final PreparedStatement getCreateTableStatement(Connection connection, TableCreateDto createDto) throws DataProcessingException { log.debug("create table columns {}", Arrays.toString(createDto.getColumns())); final StringBuilder queryBuilder = new StringBuilder() .append("CREATE TABLE ") @@ -127,25 +126,18 @@ public class PostgresService extends JdbcConnector { queryBuilder.append(");"); final String createQuery = queryBuilder.toString(); log.debug("compiled query as \"{}\"", createQuery); - return connection.prepareStatement(createQuery); - } - - private String selectStatement(Table t) { - log.debug("selecting data from {}", t.getName()); - - StringBuilder queryBuilder = new StringBuilder() - .append("SELECT "); - for (TableColumn tc : t.getColumns()) { - queryBuilder.append(tc.getInternalName() + ","); + try { + return connection.prepareStatement(createQuery); + } catch (SQLException e) { + log.error("invalid syntax: {}", e.getMessage()); + throw new DataProcessingException("invalid syntax", e); } - queryBuilder.deleteCharAt(queryBuilder.length() - 1); - queryBuilder.append(" FROM " + t.getInternalName()); - log.debug(queryBuilder.toString()); - return queryBuilder.toString(); } /** * FIXME THIS IS REMOVED IN SPRINT 2 + * Very weird ordering of arguments in the processedData for-for loop + * Why not use PostgreSQL COPY statement? * * @param processedData * @param t @@ -153,22 +145,25 @@ public class PostgresService extends JdbcConnector { */ @Override public String insertStatement(List<Map<String, Object>> processedData, Table t, List<String> headers) { - log.debug("insertStatement data into {}", t.getName()); + log.debug("insert table name: {}", t.getInternalName()); StringBuilder queryBuilder = new StringBuilder() .append("INSERT INTO ") .append(tableMapper.columnNameToString(t.getInternalName())) .append("("); for (String h : headers) { + // FIXME empty columns list in table produces nullpointer exception queryBuilder.append(t.getColumns().stream().filter(x -> x.getName().equals(h)).findFirst().get().getInternalName() + ","); } queryBuilder.deleteCharAt(queryBuilder.length() - 1); queryBuilder.append(") VALUES "); + // FIXME: no rows in processed data produce invalid syntax, but no exception thrown for (Map<String, Object> m : processedData) { queryBuilder.append("("); + // FIXME: no rows in processed data produce invalid syntax, but no exception thrown for (Map.Entry<String, Object> entry : m.entrySet()) { TableColumn tc = t.getColumns().stream().filter(x -> x.getName().equals(entry.getKey())).findFirst().get(); - if (tc.getColumnType().toString().equals("STRING")) { + if (tc.getColumnType().toString().equals("STRING") || tc.getColumnType().equals("TEXT")) { queryBuilder.append("'" + entry.getValue() + "'" + ","); } else { queryBuilder.append(entry.getValue() + ","); @@ -183,7 +178,7 @@ public class PostgresService extends JdbcConnector { return queryBuilder.toString(); } - public void deleteTable(Table table) throws DatabaseConnectionException, TableMalformedException { + public void deleteTable(Table table) throws DatabaseConnectionException, TableMalformedException, DataProcessingException { try { final PreparedStatement statement = getDeleteStatement(getConnection(table.getDatabase()), table); statement.execute(); @@ -194,12 +189,31 @@ public class PostgresService extends JdbcConnector { } @Override - final PreparedStatement getDeleteStatement(Connection connection, Table table) throws SQLException { + final PreparedStatement getDeleteStatement(Connection connection, Table table) throws DataProcessingException { final StringBuilder deleteQuery = new StringBuilder("DROP TABLE ") .append(tableMapper.columnNameToString(table.getInternalName())) .append(";"); log.debug("compiled delete table statement as {}", deleteQuery.toString()); - return connection.prepareStatement(deleteQuery.toString()); + try { + return connection.prepareStatement(deleteQuery.toString()); + } catch (SQLException e) { + log.error("invalid syntax: {}", e.getMessage()); + throw new DataProcessingException("invalid syntax", e); + } + } + + private String selectStatement(Table t) { + log.debug("selecting data from {}", t.getName()); + + StringBuilder queryBuilder = new StringBuilder() + .append("SELECT "); + for (TableColumn tc : t.getColumns()) { + queryBuilder.append(tc.getInternalName() + ","); + } + queryBuilder.deleteCharAt(queryBuilder.length() - 1); + queryBuilder.append(" FROM " + t.getInternalName()); + log.debug(queryBuilder.toString()); + return queryBuilder.toString(); } /** diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java b/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java index b3af846b0c2174dbee23183eb9c5e810b87444c2..fada3f8f12b999c4dc4cbb88fcba59ab5d59f247 100644 --- a/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java +++ b/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java @@ -69,7 +69,7 @@ public class TableService { return tables; } - public void delete(Long databaseId, Long tableId) throws TableNotFoundException, DatabaseConnectionException, TableMalformedException { + public void delete(Long databaseId, Long tableId) throws TableNotFoundException, DatabaseConnectionException, TableMalformedException, DataProcessingException { final Table table = findById(databaseId, tableId); postgresService.deleteTable(table); tableRepository.deleteById(tableId); @@ -104,7 +104,7 @@ public class TableService { @Transactional public Table create(Long databaseId, TableCreateDto createDto) throws ImageNotSupportedException, - DatabaseConnectionException, TableMalformedException, DatabaseNotFoundException { + DatabaseConnectionException, TableMalformedException, DatabaseNotFoundException, DataProcessingException { final Database database = findDatabase(databaseId); /* save in metadata db */ @@ -181,7 +181,7 @@ public class TableService { } public QueryResultDto showData(Long databaseId, Long tableId) throws ImageNotSupportedException, - DatabaseNotFoundException, TableNotFoundException, DatabaseConnectionException { + DatabaseNotFoundException, TableNotFoundException, DatabaseConnectionException, DataProcessingException { QueryResultDto queryResult = postgresService.getAllRows(findDatabase(databaseId), findById(databaseId, tableId)); for (Map<String, Object> m : queryResult.getResult()) { for (Map.Entry<String, Object> entry : m.entrySet()) {