diff --git a/.gitlab/pre-merge-test b/.gitlab/pre-merge-test
new file mode 100755
index 0000000000000000000000000000000000000000..99fae848279c4f1149ffec9dfbdf7b9f01d749f2
--- /dev/null
+++ b/.gitlab/pre-merge-test
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# DESCRIPTION: script to check if the current code base passes all tests before submitting to pipeline
+# WHEN: merge to dev, master
+
+SERVICES="container
+ database
+ discovery
+ gateway
+ query
+ table"
+
+# 1) Docker
+echo -e "\e[96m1\e[39m) Docker"
+echo "Building all"
+docker-compose build >/dev/null 2>&1
+
+if [[ $? -ne 0 ]]; then
+ echo -e "... \e[91mNOT OK\e[39m"
+else
+ echo -e "... \e[92mOK\e[39m"
+fi
+
+# 2) Maven
+echo -e "\e[96m2\e[39m) Maven"
+
+for service in $SERVICES; do
+ echo "Testing ./fda-${service}-service"
+ RESULT=$(mvn -f "./fda-${service}-service/pom.xml" clean test verify | grep -o "FAILURE")
+ if [[ $RESULT ]]; then
+ echo -e "... \e[91mNOT OK\e[39m"
+ else
+ echo -e "... \e[92mOK\e[39m"
+ fi
+done
+
+# 3) Runtime
+echo -e "\e[96m3\e[39m) Runtime"
+echo "Execute Docker runtime, look for errors"
+docker-compose up
diff --git a/docker-compose.yml b/docker-compose.yml
index 4c9736d360bcefca6000b7acebb5f04a7e0d03cd..181c7945e485ed608ad734f6067595dd6474ec3e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -206,6 +206,8 @@ services:
- fda-public
ports:
- 3000:3000
+ volumes:
+ - /tmp:/tmp
depends_on:
- fda-container-service
- fda-database-service
@@ -215,3 +217,4 @@ services:
API_CONTAINER: http://fda-container-service:9091
API_DATABASE: http://fda-database-service:9092
API_TABLES: http://fda-table-service:9094
+ API_ANALYSE: http://fda-analyse-service:5000
diff --git a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/table/CreateTableViaCsvDTO.java b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/table/CreateTableViaCsvDTO.java
deleted file mode 100644
index 76b97ca03bd377304c1a082e253f386120c267f6..0000000000000000000000000000000000000000
--- a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/table/CreateTableViaCsvDTO.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package at.tuwien.api.database.table;
-
-import lombok.*;
-
-@Setter
-@Getter
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class CreateTableViaCsvDTO {
-
- private String containerId;
-
- private String pathToFile;
-
- private char delimiter;
-
-
-}
diff --git a/fda-metadata-db/api/src/main/java/at/tuwien/api/database/table/TableCsvInformationDto.java b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/table/TableCsvInformationDto.java
new file mode 100644
index 0000000000000000000000000000000000000000..bf6063fbe1c72867fe939e28a2bed9c7ed9e52ac
--- /dev/null
+++ b/fda-metadata-db/api/src/main/java/at/tuwien/api/database/table/TableCsvInformationDto.java
@@ -0,0 +1,31 @@
+package at.tuwien.api.database.table;
+
+import at.tuwien.api.database.table.columns.ColumnTypeDto;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.*;
+
+import javax.validation.constraints.NotBlank;
+import java.util.List;
+
+@Setter
+@Getter
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class TableCsvInformationDto {
+
+ @NotBlank
+ @ApiModelProperty(name = "name", example = "Fundamentals")
+ private String name;
+
+ @NotBlank
+ @ApiModelProperty(name = "table description", required = true, example = "SEC 10K annual fillings (2016-2012) ")
+ private String description;
+
+ @NotBlank
+ private List<ColumnTypeDto> columns;
+
+ @NotBlank
+ private String fileLocation;
+
+}
diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
index 629e368b8c11bf52ae848ff272c9b337a2ec8ea1..82055f32cef4a06bd4ac2432eeae198397a69261 100644
--- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
+++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
@@ -13,9 +13,9 @@ import at.tuwien.service.QueryService;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
+import net.sf.jsqlparser.JSQLParserException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@@ -25,7 +25,7 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
-import java.sql.SQLSyntaxErrorException;
+import java.sql.SQLFeatureNotSupportedException;
import java.util.List;
import java.util.stream.Collectors;
@@ -64,7 +64,8 @@ public class QueryEndpoint {
@ApiResponse(code = 404, message = "The database does not exist."),
@ApiResponse(code = 405, message = "The container is not running."),
@ApiResponse(code = 409, message = "The container image is not supported."),})
- public ResponseEntity<?> create(@PathVariable Long id) throws ImageNotSupportedException, DatabaseConnectionException, DatabaseNotFoundException {
+ public ResponseEntity<?> create(@PathVariable Long id) throws ImageNotSupportedException,
+ DatabaseConnectionException, DatabaseNotFoundException {
queryService.create(id);
return ResponseEntity.status(HttpStatus.CREATED)
.build();
@@ -77,22 +78,22 @@ public class QueryEndpoint {
@ApiResponse(code = 404, message = "The database does not exist."),
@ApiResponse(code = 405, message = "The container is not running."),
@ApiResponse(code = 409, message = "The container image is not supported."),})
- public ResponseEntity<QueryResultDto> modify(@PathVariable Long id, @RequestBody ExecuteQueryDto dto) throws DatabaseNotFoundException, ImageNotSupportedException, SQLSyntaxErrorException {
- final QueryResultDto qr = queryService.executeStatement(id, queryMapper.queryDTOtoQuery(dto));
- return ResponseEntity.status(HttpStatus.OK)
- .contentType(MediaType.APPLICATION_JSON)
- .body(qr);
+ public ResponseEntity<QueryResultDto> modify(@PathVariable Long id, @RequestBody ExecuteQueryDto dto)
+ throws DatabaseNotFoundException, ImageNotSupportedException, SQLFeatureNotSupportedException,
+ JSQLParserException {
+ final QueryResultDto response = queryService.executeStatement(id, queryMapper.queryDTOtoQuery(dto));
+ return ResponseEntity.ok(response);
}
@PutMapping("/query/version/{timestamp}")
@ApiOperation(value = "executes a query with a given timestamp")
@ApiResponses(value = {@ApiResponse(code = 201, message = "result of Query with Timestamp")})
- public ResponseEntity<?> modify(@PathVariable Long id, @PathVariable String timestamp, @RequestBody ExecuteQueryDto dto) throws DatabaseNotFoundException, ImageNotSupportedException, SQLSyntaxErrorException {
- queryService.executeStatement(id, queryMapper.queryDTOtoQuery(dto));
- return ResponseEntity.status(HttpStatus.OK)
- .contentType(MediaType.APPLICATION_JSON)
- .build();
+ public ResponseEntity<QueryResultDto> modify(@PathVariable Long id, @PathVariable String timestamp, @RequestBody ExecuteQueryDto dto)
+ throws DatabaseNotFoundException, ImageNotSupportedException, SQLFeatureNotSupportedException,
+ JSQLParserException {
+ final QueryResultDto response = queryService.executeStatement(id, queryMapper.queryDTOtoQuery(dto));
+ return ResponseEntity.ok(response);
}
}
diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java b/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
index 2b815a9ced8d4ea00d3a4755d61270f4e171630a..a2d3ac3d8aa85f412d8703fe174b4cee3c1f0b16 100644
--- a/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
+++ b/fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
@@ -10,10 +10,18 @@ import at.tuwien.exception.ImageNotSupportedException;
import at.tuwien.exception.QueryMalformedException;
import at.tuwien.repository.DatabaseRepository;
import lombok.extern.log4j.Log4j2;
+import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.parser.CCJSqlParserManager;
+import net.sf.jsqlparser.statement.Statement;
+import net.sf.jsqlparser.statement.select.PlainSelect;
+import net.sf.jsqlparser.statement.select.Select;
+import net.sf.jsqlparser.statement.select.SelectItem;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.persistence.EntityNotFoundException;
+import java.io.StringReader;
+import java.sql.SQLFeatureNotSupportedException;
import java.sql.SQLSyntaxErrorException;
import java.sql.Timestamp;
import java.util.List;
@@ -45,11 +53,22 @@ public class QueryService {
return postgresService.getQueries(findDatabase(id));
}
- public QueryResultDto executeStatement(Long id, Query query) throws ImageNotSupportedException, DatabaseNotFoundException, SQLSyntaxErrorException {
- if (!checkValidity(query.getQuery())) {
- throw new SQLSyntaxErrorException("SQL Query contains invalid Syntax");
- }
+ public QueryResultDto executeStatement(Long id, Query query) throws ImageNotSupportedException, DatabaseNotFoundException, JSQLParserException, SQLFeatureNotSupportedException {
+ CCJSqlParserManager parserRealSql = new CCJSqlParserManager();
+
+ Statement stmt = parserRealSql.parse(new StringReader(query.getQuery()));
Database database = findDatabase(id);
+ if(stmt instanceof Select) {
+ Select selectStatement = (Select) stmt;
+ PlainSelect ps = (PlainSelect)selectStatement.getSelectBody();
+
+ List<SelectItem> selectitems = ps.getSelectItems();
+ System.out.println(ps.getFromItem().toString());
+ selectitems.stream().forEach(selectItem -> System.out.println(selectItem.toString()));
+ }
+ else {
+ throw new SQLFeatureNotSupportedException("SQL Query is not a SELECT statement - please only use SELECT statements");
+ }
saveQuery(database, query, null);
return null;
diff --git a/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
index 5736265039d11360ac8447d9dc0902bdeb8b4389..61c5509e7a424e772c46df8f953d51f1525d936e 100644
--- a/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
+++ b/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
@@ -3,6 +3,7 @@ package at.tuwien.endpoints;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.api.database.table.TableBriefDto;
import at.tuwien.api.database.table.TableCreateDto;
+import at.tuwien.api.database.table.TableCsvInformationDto;
import at.tuwien.api.database.table.TableDto;
import at.tuwien.entities.database.table.Table;
import at.tuwien.exception.*;
@@ -19,6 +20,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
+import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
@@ -74,6 +76,39 @@ public class TableEndpoint {
.body(tableMapper.tableToTableBriefDto(table));
}
+ @PostMapping("/table/csv")
+ @ApiOperation(value = "Create a table", notes = "Creates a file, which is given as a multipart file.")
+ @ApiResponses({
+ @ApiResponse(code = 201, message = "The table was created."),
+ @ApiResponse(code = 400, message = "The creation form contains invalid data."),
+ @ApiResponse(code = 401, message = "Not authorized to create a tables."),
+ @ApiResponse(code = 404, message = "The database does not exist."),
+ @ApiResponse(code = 405, message = "The container is not running."),
+ @ApiResponse(code = 409, message = "The container image is not supported."),
+ })
+ public ResponseEntity<TableDto> createViaCsv(@PathVariable("id") Long databaseId, @RequestPart("file") MultipartFile file, @RequestPart TableCsvInformationDto headers) {
+ final Table table = tableService.create(databaseId, file, headers);
+ return ResponseEntity.status(HttpStatus.CREATED)
+ .body(tableMapper.tableToTableDto(table));
+ }
+
+ @PostMapping("/table/csv/local")
+ @ApiOperation(value = "Create a table", notes = "This is done by saving a file on the shared docker filesystem and then sending the link to the file.")
+ @ApiResponses({
+ @ApiResponse(code = 201, message = "The table was created."),
+ @ApiResponse(code = 400, message = "The creation form contains invalid data."),
+ @ApiResponse(code = 401, message = "Not authorized to create a tables."),
+ @ApiResponse(code = 404, message = "The database does not exist."),
+ @ApiResponse(code = 405, message = "The container is not running."),
+ @ApiResponse(code = 409, message = "The container image is not supported."),
+ })
+ public ResponseEntity<TableDto> createViaCsv(@PathVariable("id") Long databaseId, @RequestBody TableCsvInformationDto tableCSVInformation) throws IOException {
+ final Table table = tableService.create(databaseId, tableCSVInformation);
+ return ResponseEntity.status(HttpStatus.CREATED)
+ .body(tableMapper.tableToTableDto(table));
+ }
+
+
@GetMapping("/table/{tableId}")
@ApiOperation(value = "List all tables", notes = "Lists the tables in the metadata database for this database.")
@ApiResponses({
diff --git a/fda-table-service/services/pom.xml b/fda-table-service/services/pom.xml
index 057eab57483906dfd0e2dc2615432baaecb11a85..932692ebe84f7639579cb26e94f605a89c6b0faf 100644
--- a/fda-table-service/services/pom.xml
+++ b/fda-table-service/services/pom.xml
@@ -24,6 +24,10 @@
<artifactId>super-csv</artifactId>
<version>2.4.0</version>
</dependency>
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ </dependency>
</dependencies>
<build>
@@ -52,4 +56,4 @@
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java b/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java
index c1a420ebaeb278825dd60fbea7688ab12b19f157..0141e0ff331b03992a81cc817d17f10e389853a7 100644
--- a/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java
+++ b/fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java
@@ -96,10 +96,17 @@ public class PostgresService extends JdbcConnector {
while (result.next()) {
Map<String, Object> r = new HashMap<>();
for (TableColumn tc : t.getColumns()) {
- r.put(tc.getName(), result.getString(tc.getInternalName()));
+ if (ColumnTypeDto.valueOf(tc.getColumnType()).equals(ColumnTypeDto.NUMBER)) {
+ r.put(tc.getName(), result.getDouble(tc.getInternalName()));
+ } else if (ColumnTypeDto.valueOf(tc.getColumnType()).equals(ColumnTypeDto.BOOLEAN)) {
+ r.put(tc.getName(), result.getBoolean(tc.getInternalName()));
+ } else {
+ r.put(tc.getName(), result.getString(tc.getInternalName()));
+ }
}
res.add(r);
}
+ log.debug("assembled result: {}", res);
qr.setResult(res);
return qr;
} catch (SQLException e) {
diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java b/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java
index d1e22344f9299888abc3a2b6304be25f20d47408..252bb5b71029bc0e3590410a24cea95c44ec68c8 100644
--- a/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java
+++ b/fda-table-service/services/src/main/java/at/tuwien/service/TableService.java
@@ -2,6 +2,8 @@ package at.tuwien.service;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.api.database.table.TableCreateDto;
+import at.tuwien.api.database.table.TableCsvInformationDto;
+import at.tuwien.api.database.table.columns.ColumnCreateDto;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.table.Table;
import at.tuwien.entities.database.table.columns.TableColumn;
@@ -11,6 +13,7 @@ import at.tuwien.repository.DatabaseRepository;
import at.tuwien.repository.TableRepository;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.mock.web.MockMultipartFile;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.supercsv.cellprocessor.constraint.NotNull;
@@ -24,6 +27,9 @@ import javax.transaction.Transactional;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -177,14 +183,83 @@ public class TableService {
}
}
+ private String[] readHeader(MultipartFile file) throws IOException {
+ ICsvMapReader mapReader = null;
+ try {
+ Reader reader = new InputStreamReader(file.getInputStream());
+ mapReader = new CsvMapReader(reader, CsvPreference.STANDARD_PREFERENCE);
+
+ String[] header = mapReader.getHeader(true);
+ return header;
+
+ } catch(IOException e) {
+ e.printStackTrace();
+ } finally {
+ if( mapReader != null ) {
+ mapReader.close();
+ }
+ }
+ return null;
+ }
+
+ // TODO ms what is this for? It does ony print to stdout
public QueryResultDto showData(Long databaseId, Long tableId) throws ImageNotSupportedException,
DatabaseNotFoundException, TableNotFoundException, DatabaseConnectionException, DataProcessingException {
QueryResultDto queryResult = postgresService.getAllRows(findDatabase(databaseId), findById(databaseId, tableId));
- for (Map<String, Object> m : queryResult.getResult()) {
+ for (Map<String, Object> m : queryResult.getResult() ) {
for (Map.Entry<String, Object> entry : m.entrySet()) {
- System.out.print(entry.getKey() + ": " + entry.getValue() + ", ");
+ log.debug("{}: {}", entry.getKey(), entry.getValue());
}
}
return queryResult;
}
+
+ public Table create(Long databaseId, MultipartFile file, TableCsvInformationDto tableCSVInformation) {
+ try {
+ String[] header = readHeader(file);
+ log.debug("table csv info: {}", tableCSVInformation);
+ TableCreateDto tcd = new TableCreateDto();
+ tcd.setName(tableCSVInformation.getName());
+ tcd.setDescription(tableCSVInformation.getDescription());
+ ColumnCreateDto[] cdtos = new ColumnCreateDto[header.length];
+ log.debug("header: {}", header);
+ for (int i = 0; i < header.length; i++) {
+ ColumnCreateDto c = new ColumnCreateDto();
+ c.setName(header[i]);
+ c.setType(tableCSVInformation.getColumns().get(i));
+ c.setNullAllowed(true);
+ //TODO FIX THAT not only id is primary key
+ if(header[i].equals("id")) {
+ c.setPrimaryKey(true);
+ } else {
+ c.setPrimaryKey(false);
+ }
+ cdtos[i] = c;
+ }
+ tcd.setColumns(cdtos);
+ Table table = create(databaseId, tcd);
+ QueryResultDto insert = insert(databaseId, table.getId(), file);
+ return table;
+ } catch (Exception e) {
+ e.printStackTrace();
+ log.error(e.getMessage());
+ }
+ return null;
+ }
+
+ public Table create(Long databaseId, TableCsvInformationDto tableCSVInformation) throws IOException {
+ Path path = Paths.get("/tmp/" + tableCSVInformation.getFileLocation());
+ String contentType = "multipart/form-data";
+ byte[] content = null;
+ try {
+ content = Files.readAllBytes(path);
+ } catch (final IOException e) {
+ }
+ MultipartFile multipartFile = new MockMultipartFile(tableCSVInformation.getFileLocation(),
+ tableCSVInformation.getFileLocation(), contentType, content);
+ Files.deleteIfExists(path);
+ return create(databaseId, multipartFile,tableCSVInformation);
+ }
+
+
}
diff --git a/fda-ui/.env b/fda-ui/.env
index 9e4531c8a85705c4e7b9b0e5ffcea34b37dbb34a..05057611a6ac1a8e0c5195efea37f086f99b9285 100644
--- a/fda-ui/.env
+++ b/fda-ui/.env
@@ -2,3 +2,4 @@
API_CONTAINER="http://localhost:9091"
API_DATABASE="http://localhost:9092"
API_TABLES="http://localhost:9094"
+API_ANALYSE="http://localhost:5000"
diff --git a/fda-ui/Dockerfile b/fda-ui/Dockerfile
index f19851acb36f66ed94c3ad2ba9799009dc446eee..cd22f0bca67342f987c559e303d25ec1833fb949 100644
--- a/fda-ui/Dockerfile
+++ b/fda-ui/Dockerfile
@@ -14,6 +14,7 @@ COPY ./components ./components
COPY ./lang ./lang
COPY ./layouts ./layouts
COPY ./middleware ./middleware
+COPY ./server-middleware ./server-middleware
COPY ./pages ./pages
COPY ./plugins ./plugins
COPY ./store ./store
diff --git a/fda-ui/components/TableCreate.vue b/fda-ui/components/TableCreate.vue
index ce0e48fdae388c6c4d01e0aae38b46a7022d7ba6..8012b2c94e78b61f31d36833eaf6a38cacad44ee 100644
--- a/fda-ui/components/TableCreate.vue
+++ b/fda-ui/components/TableCreate.vue
@@ -1,56 +1,58 @@
<template>
- <v-card>
- <v-card-title class="pb-0">
- Create Table
- </v-card-title>
- <!-- <v-card-subtitle>
- Table is not created until the "Create Table" button is pressed.
- </v-card-subtitle> -->
- <v-card-text>
- <v-text-field
- v-model="name"
- label="Table Name"
- :rules="[v => !!v || $t('Required')]"
- required />
- <v-text-field
- v-model="description"
- label="Description" />
- <v-btn @click="addColumn">
- Add Column
- </v-btn>
- </v-card-text>
- <v-card-text v-for="(c, idx) in columns" :key="idx" class="pa-3">
- <v-row class="column pa-2 ml-1 mr-1">
- <v-col cols="4">
- <v-text-field v-model="c.name" required label="Name" />
- </v-col>
- <v-col cols="3">
- <v-select
- v-model="c.type"
- :items="columnTypes"
- item-value="value"
- required
- label="Data Type" />
- </v-col>
- <v-col cols="2">
- <v-checkbox v-model="c.primaryKey" label="Primary Key" />
- </v-col>
- <v-col cols="2">
- <v-checkbox v-model="c.nullAllowed" label="Null Allowed" />
- </v-col>
+ <div>
+ <v-card>
+ <v-card-title class="pb-0">
+ Create Table
+ </v-card-title>
+ <!-- <v-card-subtitle>
+ Table is not created until the "Create Table" button is pressed.
+ </v-card-subtitle> -->
+ <v-card-text>
+ <v-text-field
+ v-model="name"
+ label="Table Name"
+ :rules="[v => !!v || $t('Required')]"
+ required />
+ <v-text-field
+ v-model="description"
+ label="Description" />
+ <v-btn @click="addColumn">
+ Add Column
+ </v-btn>
+ </v-card-text>
+ <v-card-text v-for="(c, idx) in columns" :key="idx" class="pa-3">
+ <v-row class="column pa-2 ml-1 mr-1">
+ <v-col cols="4">
+ <v-text-field v-model="c.name" required label="Name" />
+ </v-col>
+ <v-col cols="3">
+ <v-select
+ v-model="c.type"
+ :items="columnTypes"
+ item-value="value"
+ required
+ label="Data Type" />
+ </v-col>
+ <v-col cols="2">
+ <v-checkbox v-model="c.primaryKey" label="Primary Key" />
+ </v-col>
+ <v-col cols="2">
+ <v-checkbox v-model="c.nullAllowed" label="Null Allowed" />
+ </v-col>
+ <v-spacer />
+ <v-btn title="Remove column" outlined icon @click="removeColumn(idx)">
+ <v-icon>mdi-minus</v-icon>
+ </v-btn>
+ </v-row>
+ </v-card-text>
+ <v-card-actions>
<v-spacer />
- <v-btn title="Remove column" outlined icon @click="removeColumn(idx)">
- <v-icon>mdi-minus</v-icon>
+ <v-btn :disabled="!canCreateTable()" @click="createTable">
+ Create Table
</v-btn>
- </v-row>
- </v-card-text>
- <v-card-actions>
- <v-spacer />
- <v-btn :disabled="!canCreateTable()" @click="createTable">
- Create Table
- </v-btn>
- </v-card-actions>
- </v-card>
+ </v-card-actions>
+ </v-card>
+ </div>
</template>
<script>
@@ -98,13 +100,7 @@ export default {
const data = {
name: this.name,
description: this.description,
- columns: this.columns.map((c) => {
- // c.nullAllowed = c.isNullAllowed
- // c.primaryKey = c.isPrimaryKey
- // delete c.isPrimaryKey
- // delete c.isNullAllowed
- return c
- })
+ columns: this.columns
}
try {
const res = await this.$axios.post(`/api/tables/api/database/${this.$route.params.db_id}/table`, data)
diff --git a/fda-ui/nuxt.config.js b/fda-ui/nuxt.config.js
index f5065166aa6cbedfef3a38a3cfa68be62724fa2d..cbfeeb19accc05e359de03231c92a21b4d7545a3 100644
--- a/fda-ui/nuxt.config.js
+++ b/fda-ui/nuxt.config.js
@@ -1,3 +1,4 @@
+import path from 'path'
import colors from 'vuetify/es5/util/colors'
import isDocker from 'is-docker'
@@ -76,9 +77,14 @@ export default {
proxy: {
'/api/container': process.env.API_CONTAINER,
'/api/database': process.env.API_DATABASE,
+ '/api/analyse': process.env.API_ANALYSE,
'/api/tables': { target: process.env.API_TABLES, pathRewrite: { '^/api/tables/': '' } }
},
+ serverMiddleware: [
+ { path: '/server-middleware', handler: path.resolve(__dirname, 'server-middleware/index.js') }
+ ],
+
// Vuetify module configuration (https://go.nuxtjs.dev/config-vuetify)
vuetify: {
customVariables: ['~/assets/variables.scss'],
diff --git a/fda-ui/package.json b/fda-ui/package.json
index b12d61ba371c2df59cd046579b6c514c42d0bcd4..5a019acca08e9ca389f1c4ab8eb7b8cb4518bf2d 100644
--- a/fda-ui/package.json
+++ b/fda-ui/package.json
@@ -3,7 +3,7 @@
"version": "1.0.0",
"private": true,
"scripts": {
- "dev": "nuxt",
+ "dev": "nuxt --port 3001",
"docker": "nuxt > /dev/null",
"build": "nuxt build",
"start": "nuxt start",
@@ -22,6 +22,8 @@
"core-js": "^3.6.5",
"date-fns": "^2.16.1",
"is-docker": "^2.2.1",
+ "multer": "^1.4.2",
+ "node-fetch": "^2.6.1",
"nuxt": "^2.12.2",
"nuxt-i18n": "^6.15.4",
"vue-toast-notification": "^0.5.4",
diff --git a/fda-ui/pages/db/_db_id/tables/index.vue b/fda-ui/pages/db/_db_id/tables/index.vue
index 492567c52f72ad86e1b6b99ef22b0508243747b3..f6d687c77e972a536ec22fc8fa4c64d56621ef8d 100644
--- a/fda-ui/pages/db/_db_id/tables/index.vue
+++ b/fda-ui/pages/db/_db_id/tables/index.vue
@@ -5,6 +5,13 @@
</h3>
<TableList />
<TableCreate />
+ <v-card class="mt-1">
+ <v-card-text>
+ <nuxt-link class="table_from_csv" :to="`/db/${$route.params.db_id}/tables/table_from_csv`">
+ Create table from CSV file
+ </nuxt-link>
+ </v-card-text>
+ </v-card>
</div>
</template>
<script>
@@ -27,5 +34,8 @@ export default {
}
</script>
-<style>
+<style scoped>
+a.table_from_csv {
+ font-size: 14pt;
+}
</style>
diff --git a/fda-ui/pages/db/_db_id/tables/table_from_csv.vue b/fda-ui/pages/db/_db_id/tables/table_from_csv.vue
new file mode 100644
index 0000000000000000000000000000000000000000..5140c075fde9da012aa3a410ab1ea79662e6f108
--- /dev/null
+++ b/fda-ui/pages/db/_db_id/tables/table_from_csv.vue
@@ -0,0 +1,166 @@
+<template>
+ <div>
+ <h3 class="mb-2 mt-1">Table from CSV</h3>
+ <v-stepper v-model="step" vertical>
+ <v-stepper-step :complete="step > 1" step="1">
+ Table
+ </v-stepper-step>
+
+ <v-stepper-content class="pt-0 pb-1" step="1">
+ <v-text-field v-model="tableName" required label="Name" />
+ <v-text-field v-model="tableDesc" label="Description" />
+ <v-btn :disabled="!step1Valid" color="primary" @click="step = 2">
+ Continue
+ </v-btn>
+ </v-stepper-content>
+
+ <v-stepper-step :complete="step > 2" step="2">
+ Upload CSV file
+ </v-stepper-step>
+
+ <v-stepper-content step="2">
+ <v-row dense>
+ <v-col cols="8">
+ <v-file-input
+ v-model="file"
+ accept="text/csv"
+ show-size
+ label="CSV File" />
+ </v-col>
+ <v-col cols="4" class="mt-3">
+ <v-btn :disabled="!file" :loading="loading" @click="upload">Upload</v-btn>
+ </v-col>
+ </v-row>
+ </v-stepper-content>
+
+ <v-stepper-step :complete="step > 3" step="3">
+ Choose data type of columns
+ </v-stepper-step>
+ <v-stepper-content step="3">
+ <div v-for="(c, idx) in columns" :key="idx">
+ <v-row dense class="column pa-2 ml-1 mr-1 mb-2">
+ <v-col cols="4">
+ <v-text-field v-model="c.name" disabled required label="Name" />
+ </v-col>
+ <v-col cols="3">
+ <v-select
+ v-model="c.type"
+ :items="columnTypes"
+ item-value="value"
+ required
+ label="Data Type" />
+ </v-col>
+ <v-col cols="auto" class="pl-2">
+ <v-checkbox v-model="c.primaryKey" label="Primary Key" />
+ </v-col>
+ <v-col cols="auto" class="pl-10">
+ <v-checkbox v-model="c.nullAllowed" label="Null Allowed" />
+ </v-col>
+ </v-row>
+ </div>
+
+ <v-btn class="mt-2" color="primary" @click="createTable">
+ Continue
+ </v-btn>
+ </v-stepper-content>
+
+ <v-stepper-step
+ :complete="step > 4"
+ step="4">
+ Done
+ </v-stepper-step>
+
+ <v-stepper-content step="4">
+ Proceed to table view.
+ <div class="mt-2">
+ <v-btn :to="`/db/${$route.params.db_id}/tables/${newTableId}`" outlined>
+ <v-icon>mdi-table</v-icon>
+ View
+ </v-btn>
+ </div>
+ </v-stepper-content>
+ </v-stepper>
+ </div>
+</template>
+<script>
+export default {
+ name: 'TableFromCSV',
+ components: {
+ },
+ data () {
+ return {
+ step: 1,
+ tableName: '',
+ tableDesc: '',
+ loading: false,
+ file: null,
+ fileLocation: null,
+ columns: [],
+ columnTypes: [
+ { value: 'ENUM', text: 'ENUM' },
+ { value: 'BOOLEAN', text: 'BOOLEAN' },
+ { value: 'NUMBER', text: 'NUMBER' },
+ { value: 'BLOB', text: 'BLOB' },
+ { value: 'DATE', text: 'DATE' },
+ { value: 'STRING', text: 'STRING' },
+ { value: 'TEXT', text: 'TEXT' }
+ ],
+ newTableId: 42
+ }
+ },
+ computed: {
+ step1Valid () {
+ return this.tableName.length
+ }
+ },
+ mounted () {
+ },
+ methods: {
+ async upload () {
+ this.loading = true
+ const url = '/server-middleware/table_from_csv'
+ const data = new FormData()
+ data.append('file', this.file)
+ try {
+ const res = await this.$axios.post(url, data, {
+ headers: { 'Content-Type': 'multipart/form-data' }
+ })
+ if (res.data.success) {
+ this.columns = res.data.columns
+ this.fileLocation = res.data.file.filename
+ this.step = 3
+ } else {
+ this.$toast.error('Could not upload CSV data')
+ }
+ } catch (err) {
+ this.$toast.error('Could not upload data.')
+ }
+ this.loading = false
+ },
+ async createTable () {
+ const url = `/api/tables/api/database/${this.$route.params.db_id}/table/csv/local`
+ const data = {
+ columns: this.columns.map(c => c.type),
+ description: this.tableDesc,
+ name: this.tableName,
+ fileLocation: this.fileLocation
+ }
+ let res
+ try {
+ res = await this.$axios.post(url, data)
+ this.newTableId = res.data.id
+ } catch (err) {
+ console.log(err)
+ }
+ if (res && res.data && res.data.id) {
+ this.step = 4
+ } else {
+ this.$toast.error('Could not create table.')
+ }
+ }
+ }
+}
+</script>
+
+<style scoped>
+</style>
diff --git a/fda-ui/server-middleware/index.js b/fda-ui/server-middleware/index.js
index e6bcce7d4c7740ae1ce97b500db34f31524faf52..94eabc512ac82582de6fd2b7e93fca3f41b76e0c 100644
--- a/fda-ui/server-middleware/index.js
+++ b/fda-ui/server-middleware/index.js
@@ -1,13 +1,48 @@
-export default function (req, res, next) {
- // req is the Node.js http request object
- console.log(req.url)
+// const bodyParser = require('body-parser')
+const app = require('express')()
+const multer = require('multer')
+const upload = multer({ dest: '/tmp' })
+const fetch = require('node-fetch')
- // res is the Node.js http response object
+// TODO extend me
+const colTypeMap = {
+ Boolean: 'BOOLEAN',
+ Date: 'DATE',
+ Integer: 'NUMBER',
+ Numeric: 'NUMBER',
+ String: 'STRING',
+ Timestamp: 'DATE'
+}
- // next is a function to call to invoke the next middleware
- // Don't forget to call next at the end if your middleware is not an endpoint!
- // next()
- console.log(res)
+app.post('/table_from_csv', upload.single('file'), async (req, res) => {
+ const { file } = req
+ const { path } = file
- return res.json({ hi: 'foo' })
-}
+ // send path to analyse service
+ let analysis
+ try {
+ analysis = await fetch(`${process.env.API_ANALYSE}/datatypesbypath?filepath=${path}`)
+ analysis = await analysis.json()
+ } catch (error) {
+ return res.json({ success: false, error })
+ }
+
+ // map messytables / CoMi's `determine_dt` column types to ours
+ // e.g. "Integer" -> "NUMBER"
+ let entries = Object.entries(analysis.columns)
+ entries = entries.map(([k, v]) => {
+ if (colTypeMap[v]) {
+ v = colTypeMap[v]
+ }
+ return {
+ name: k,
+ type: v,
+ nullAllowed: true,
+ primaryKey: false
+ }
+ })
+
+ res.json({ success: true, file, columns: entries })
+})
+
+module.exports = app
diff --git a/fda-ui/yarn.lock b/fda-ui/yarn.lock
index dd67cc0b1f649aaac4ebc0aa20687ac86f2e5614..830b0ea1598ca74a15c03b90cf6ddf03b813d76b 100644
--- a/fda-ui/yarn.lock
+++ b/fda-ui/yarn.lock
@@ -1682,6 +1682,11 @@ anymatch@~3.1.1:
normalize-path "^3.0.0"
picomatch "^2.0.4"
+append-field@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/append-field/-/append-field-1.0.0.tgz#1e3440e915f0b1203d23748e78edd7b9b5b43e56"
+ integrity sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY=
+
aproba@^1.1.1:
version "1.2.0"
resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz"
@@ -2114,6 +2119,14 @@ builtin-status-codes@^3.0.0:
resolved "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz"
integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=
+busboy@^0.2.11:
+ version "0.2.14"
+ resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.2.14.tgz#6c2a622efcf47c57bbbe1e2a9c37ad36c7925453"
+ integrity sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=
+ dependencies:
+ dicer "0.2.5"
+ readable-stream "1.1.x"
+
bytes@3.0.0:
version "3.0.0"
resolved "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz"
@@ -2516,7 +2529,7 @@ concat-map@0.0.1:
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz"
integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=
-concat-stream@^1.5.0:
+concat-stream@^1.5.0, concat-stream@^1.5.2:
version "1.6.2"
resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz"
integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==
@@ -3007,6 +3020,14 @@ detect-libc@^1.0.3:
resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz"
integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=
+dicer@0.2.5:
+ version "0.2.5"
+ resolved "https://registry.yarnpkg.com/dicer/-/dicer-0.2.5.tgz#5996c086bb33218c812c090bddc09cd12facb70f"
+ integrity sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=
+ dependencies:
+ readable-stream "1.1.x"
+ streamsearch "0.1.2"
+
diffie-hellman@^5.0.0:
version "5.0.3"
resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz"
@@ -4681,6 +4702,11 @@ is-wsl@^1.1.0:
resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-1.1.0.tgz"
integrity sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=
+isarray@0.0.1:
+ version "0.0.1"
+ resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
+ integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=
+
isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz"
@@ -5264,6 +5290,20 @@ ms@2.1.2:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz"
integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
+multer@^1.4.2:
+ version "1.4.2"
+ resolved "https://registry.yarnpkg.com/multer/-/multer-1.4.2.tgz#2f1f4d12dbaeeba74cb37e623f234bf4d3d2057a"
+ integrity sha512-xY8pX7V+ybyUpbYMxtjM9KAiD9ixtg5/JkeKUTD6xilfDv0vzzOFcCp4Ljb1UU3tSOM3VTZtKo63OmzOrGi3Cg==
+ dependencies:
+ append-field "^1.0.0"
+ busboy "^0.2.11"
+ concat-stream "^1.5.2"
+ mkdirp "^0.5.1"
+ object-assign "^4.1.1"
+ on-finished "^2.3.0"
+ type-is "^1.6.4"
+ xtend "^4.0.0"
+
multimap@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/multimap/-/multimap-1.1.0.tgz"
@@ -6725,6 +6765,16 @@ read-pkg@^5.2.0:
string_decoder "~1.1.1"
util-deprecate "~1.0.1"
+readable-stream@1.1.x:
+ version "1.1.14"
+ resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9"
+ integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk=
+ dependencies:
+ core-util-is "~1.0.0"
+ inherits "~2.0.1"
+ isarray "0.0.1"
+ string_decoder "~0.10.x"
+
readable-stream@^3.1.1, readable-stream@^3.6.0:
version "3.6.0"
resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz"
@@ -7371,6 +7421,11 @@ stream-shift@^1.0.0:
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz"
integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==
+streamsearch@0.1.2:
+ version "0.1.2"
+ resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"
+ integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=
+
strict-uri-encode@^1.0.0:
version "1.1.0"
resolved "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz"
@@ -7425,6 +7480,11 @@ string_decoder@^1.0.0, string_decoder@^1.1.1:
dependencies:
safe-buffer "~5.2.0"
+string_decoder@~0.10.x:
+ version "0.10.31"
+ resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
+ integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=
+
string_decoder@~1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz"
@@ -7747,7 +7807,7 @@ type-fest@^0.8.1:
resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz"
integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==
-type-is@~1.6.17, type-is@~1.6.18:
+type-is@^1.6.4, type-is@~1.6.17, type-is@~1.6.18:
version "1.6.18"
resolved "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz"
integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==