Merge branch 'ui-sprint-2' into dev

 Conflicts:
	fda-metadata-db/api/src/main/java/at/tuwien/api/database/table/TableCSVInformation.java
	fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
	fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
	fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
	fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java
	fda-table-service/services/src/main/java/at/tuwien/service/TableService.java
	fda-ui/Dockerfile


Former-commit-id: 2e6ddfc4

.gitlab/pre-merge-test

+#!/bin/bash
+
+# DESCRIPTION: script to check if the current code base passes all tests before submitting to pipeline
+# WHEN: merge to dev, master
+
+SERVICES="container
+          database
+          discovery
+          gateway
+          query
+          table"
+
+# 1) Docker
+echo -e "\e[96m1\e[39m) Docker"
+echo "Building all"
+docker-compose build >/dev/null 2>&1
+
+if [[ $? -ne 0 ]]; then
+  echo -e "... \e[91mNOT OK\e[39m"
+else
+  echo -e "... \e[92mOK\e[39m"
+fi
+
+# 2) Maven
+echo -e "\e[96m2\e[39m) Maven"
+
+for service in $SERVICES; do
+  echo "Testing ./fda-${service}-service"
+  RESULT=$(mvn -f "./fda-${service}-service/pom.xml" clean test verify | grep -o "FAILURE")
+  if [[ $RESULT ]]; then
+    echo -e "... \e[91mNOT OK\e[39m"
+  else
+    echo -e "... \e[92mOK\e[39m"
+  fi
+done
+
+# 3) Runtime
+echo -e "\e[96m3\e[39m) Runtime"
+echo "Execute Docker runtime, look for errors"
+docker-compose up

docker-compose.yml

@@ -206,6 +206,8 @@ services:
       - fda-public
     ports:
       - 3000:3000
+    volumes:
+      - /tmp:/tmp
     depends_on:
       - fda-container-service
       - fda-database-service
@@ -215,3 +217,4 @@ services:
       API_CONTAINER: http://fda-container-service:9091
       API_DATABASE: http://fda-database-service:9092
       API_TABLES: http://fda-table-service:9094
+      API_ANALYSE: http://fda-analyse-service:5000

fda-metadata-db/api/src/main/java/at/tuwien/api/database/table/CreateTableViaCsvDTO.java

-package at.tuwien.api.database.table;
-
-import lombok.*;
-
-@Setter
-@Getter
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class CreateTableViaCsvDTO {
-
-    private String containerId;
-
-    private String pathToFile;
-
-    private char delimiter;
-
-
-}

fda-metadata-db/api/src/main/java/at/tuwien/api/database/table/TableCsvInformationDto.java

+package at.tuwien.api.database.table;
+
+import at.tuwien.api.database.table.columns.ColumnTypeDto;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.*;
+
+import javax.validation.constraints.NotBlank;
+import java.util.List;
+
+@Setter
+@Getter
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class TableCsvInformationDto {
+
+    @NotBlank
+    @ApiModelProperty(name = "name", example = "Fundamentals")
+    private String name;
+
+    @NotBlank
+    @ApiModelProperty(name = "table description", required = true, example = "SEC 10K annual fillings (2016-2012) ")
+    private String description;
+
+    @NotBlank
+    private List<ColumnTypeDto> columns;
+
+    @NotBlank
+    private String fileLocation;
+
+}

fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java

@@ -13,9 +13,9 @@ import at.tuwien.service.QueryService;
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiResponse;
 import io.swagger.annotations.ApiResponses;
+import net.sf.jsqlparser.JSQLParserException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -25,7 +25,7 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
-import java.sql.SQLSyntaxErrorException;
+import java.sql.SQLFeatureNotSupportedException;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -64,7 +64,8 @@ public class QueryEndpoint {
             @ApiResponse(code = 404, message = "The database does not exist."),
             @ApiResponse(code = 405, message = "The container is not running."),
             @ApiResponse(code = 409, message = "The container image is not supported."),})
-    public ResponseEntity<?> create(@PathVariable Long id) throws ImageNotSupportedException, DatabaseConnectionException, DatabaseNotFoundException {
+    public ResponseEntity<?> create(@PathVariable Long id) throws ImageNotSupportedException,
+            DatabaseConnectionException, DatabaseNotFoundException {
         queryService.create(id);
         return ResponseEntity.status(HttpStatus.CREATED)
                 .build();
@@ -77,22 +78,22 @@ public class QueryEndpoint {
             @ApiResponse(code = 404, message = "The database does not exist."),
             @ApiResponse(code = 405, message = "The container is not running."),
             @ApiResponse(code = 409, message = "The container image is not supported."),})
-    public ResponseEntity<QueryResultDto> modify(@PathVariable Long id, @RequestBody ExecuteQueryDto dto) throws DatabaseNotFoundException, ImageNotSupportedException, SQLSyntaxErrorException {
-        final QueryResultDto qr = queryService.executeStatement(id, queryMapper.queryDTOtoQuery(dto));
-        return ResponseEntity.status(HttpStatus.OK)
-                .contentType(MediaType.APPLICATION_JSON)
-                .body(qr);
+    public ResponseEntity<QueryResultDto> modify(@PathVariable Long id, @RequestBody ExecuteQueryDto dto)
+            throws DatabaseNotFoundException, ImageNotSupportedException, SQLFeatureNotSupportedException,
+            JSQLParserException {
+        final QueryResultDto response = queryService.executeStatement(id, queryMapper.queryDTOtoQuery(dto));
+        return ResponseEntity.ok(response);
     }
 
 
     @PutMapping("/query/version/{timestamp}")
     @ApiOperation(value = "executes a query with a given timestamp")
     @ApiResponses(value = {@ApiResponse(code = 201, message = "result of Query with Timestamp")})
-    public ResponseEntity<?> modify(@PathVariable Long id, @PathVariable String timestamp, @RequestBody ExecuteQueryDto dto) throws DatabaseNotFoundException, ImageNotSupportedException, SQLSyntaxErrorException {
-        queryService.executeStatement(id, queryMapper.queryDTOtoQuery(dto));
-        return ResponseEntity.status(HttpStatus.OK)
-                .contentType(MediaType.APPLICATION_JSON)
-                .build();
+    public ResponseEntity<QueryResultDto> modify(@PathVariable Long id, @PathVariable String timestamp, @RequestBody ExecuteQueryDto dto)
+            throws DatabaseNotFoundException, ImageNotSupportedException, SQLFeatureNotSupportedException,
+            JSQLParserException {
+        final QueryResultDto response = queryService.executeStatement(id, queryMapper.queryDTOtoQuery(dto));
+        return ResponseEntity.ok(response);
     }
 
 }

fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java

@@ -10,10 +10,18 @@ import at.tuwien.exception.ImageNotSupportedException;
 import at.tuwien.exception.QueryMalformedException;
 import at.tuwien.repository.DatabaseRepository;
 import lombok.extern.log4j.Log4j2;
+import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.parser.CCJSqlParserManager;
+import net.sf.jsqlparser.statement.Statement;
+import net.sf.jsqlparser.statement.select.PlainSelect;
+import net.sf.jsqlparser.statement.select.Select;
+import net.sf.jsqlparser.statement.select.SelectItem;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
 import javax.persistence.EntityNotFoundException;
+import java.io.StringReader;
+import java.sql.SQLFeatureNotSupportedException;
 import java.sql.SQLSyntaxErrorException;
 import java.sql.Timestamp;
 import java.util.List;
@@ -45,11 +53,22 @@ public class QueryService {
         return postgresService.getQueries(findDatabase(id));
     }
 
-    public QueryResultDto executeStatement(Long id, Query query) throws ImageNotSupportedException, DatabaseNotFoundException, SQLSyntaxErrorException {
-        if (!checkValidity(query.getQuery())) {
-            throw new SQLSyntaxErrorException("SQL Query contains invalid Syntax");
-        }
+    public QueryResultDto executeStatement(Long id, Query query) throws ImageNotSupportedException, DatabaseNotFoundException, JSQLParserException, SQLFeatureNotSupportedException {
+        CCJSqlParserManager parserRealSql = new CCJSqlParserManager();
+
+        Statement stmt = parserRealSql.parse(new StringReader(query.getQuery()));
         Database database = findDatabase(id);
+        if(stmt instanceof Select) {
+            Select selectStatement = (Select) stmt;
+            PlainSelect ps = (PlainSelect)selectStatement.getSelectBody();
+
+            List<SelectItem> selectitems = ps.getSelectItems();
+            System.out.println(ps.getFromItem().toString());
+            selectitems.stream().forEach(selectItem -> System.out.println(selectItem.toString()));
+        }
+        else {
+            throw new SQLFeatureNotSupportedException("SQL Query is not a SELECT statement - please only use SELECT statements");
+        }
         saveQuery(database, query, null);
 
         return null;

fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java

@@ -3,6 +3,7 @@ package at.tuwien.endpoints;
 import at.tuwien.api.database.query.QueryResultDto;
 import at.tuwien.api.database.table.TableBriefDto;
 import at.tuwien.api.database.table.TableCreateDto;
+import at.tuwien.api.database.table.TableCsvInformationDto;
 import at.tuwien.api.database.table.TableDto;
 import at.tuwien.entities.database.table.Table;
 import at.tuwien.exception.*;
@@ -19,6 +20,7 @@ import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -74,6 +76,39 @@ public class TableEndpoint {
                 .body(tableMapper.tableToTableBriefDto(table));
     }
 
+    @PostMapping("/table/csv")
+    @ApiOperation(value = "Create a table", notes = "Creates a file, which is given as a multipart file.")
+    @ApiResponses({
+            @ApiResponse(code = 201, message = "The table was created."),
+            @ApiResponse(code = 400, message = "The creation form contains invalid data."),
+            @ApiResponse(code = 401, message = "Not authorized to create a tables."),
+            @ApiResponse(code = 404, message = "The database does not exist."),
+            @ApiResponse(code = 405, message = "The container is not running."),
+            @ApiResponse(code = 409, message = "The container image is not supported."),
+    })
+    public ResponseEntity<TableDto> createViaCsv(@PathVariable("id") Long databaseId, @RequestPart("file") MultipartFile file, @RequestPart TableCsvInformationDto headers) {
+        final Table table = tableService.create(databaseId, file, headers);
+        return ResponseEntity.status(HttpStatus.CREATED)
+                .body(tableMapper.tableToTableDto(table));
+    }
+
+    @PostMapping("/table/csv/local")
+    @ApiOperation(value = "Create a table", notes = "This is done by saving a file on the shared docker filesystem and then sending the link to the file.")
+    @ApiResponses({
+            @ApiResponse(code = 201, message = "The table was created."),
+            @ApiResponse(code = 400, message = "The creation form contains invalid data."),
+            @ApiResponse(code = 401, message = "Not authorized to create a tables."),
+            @ApiResponse(code = 404, message = "The database does not exist."),
+            @ApiResponse(code = 405, message = "The container is not running."),
+            @ApiResponse(code = 409, message = "The container image is not supported."),
+    })
+    public ResponseEntity<TableDto> createViaCsv(@PathVariable("id") Long databaseId, @RequestBody TableCsvInformationDto tableCSVInformation) throws IOException {
+        final Table table = tableService.create(databaseId, tableCSVInformation);
+        return ResponseEntity.status(HttpStatus.CREATED)
+                .body(tableMapper.tableToTableDto(table));
+    }
+
+
     @GetMapping("/table/{tableId}")
     @ApiOperation(value = "List all tables", notes = "Lists the tables in the metadata database for this database.")
     @ApiResponses({

fda-table-service/services/pom.xml

@@ -24,6 +24,10 @@
             <artifactId>super-csv</artifactId>
             <version>2.4.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-test</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

fda-table-service/services/src/main/java/at/tuwien/service/PostgresService.java

@@ -96,10 +96,17 @@ public class PostgresService extends JdbcConnector {
             while (result.next()) {
                 Map<String, Object> r = new HashMap<>();
                 for (TableColumn tc : t.getColumns()) {
+                    if (ColumnTypeDto.valueOf(tc.getColumnType()).equals(ColumnTypeDto.NUMBER)) {
+                        r.put(tc.getName(), result.getDouble(tc.getInternalName()));
+                    } else if (ColumnTypeDto.valueOf(tc.getColumnType()).equals(ColumnTypeDto.BOOLEAN)) {
+                        r.put(tc.getName(), result.getBoolean(tc.getInternalName()));
+                    } else {
                         r.put(tc.getName(), result.getString(tc.getInternalName()));
                     }
+                }
                 res.add(r);
             }
+            log.debug("assembled result: {}", res);
             qr.setResult(res);
             return qr;
         } catch (SQLException e) {

fda-table-service/services/src/main/java/at/tuwien/service/TableService.java

@@ -2,6 +2,8 @@ package at.tuwien.service;
 
 import at.tuwien.api.database.query.QueryResultDto;
 import at.tuwien.api.database.table.TableCreateDto;
+import at.tuwien.api.database.table.TableCsvInformationDto;
+import at.tuwien.api.database.table.columns.ColumnCreateDto;
 import at.tuwien.entities.database.Database;
 import at.tuwien.entities.database.table.Table;
 import at.tuwien.entities.database.table.columns.TableColumn;
@@ -11,6 +13,7 @@ import at.tuwien.repository.DatabaseRepository;
 import at.tuwien.repository.TableRepository;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.mock.web.MockMultipartFile;
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 import org.supercsv.cellprocessor.constraint.NotNull;
@@ -24,6 +27,9 @@ import javax.transaction.Transactional;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -177,14 +183,83 @@ public class TableService {
         }
     }
 
+    private String[] readHeader(MultipartFile file) throws IOException {
+        ICsvMapReader mapReader = null;
+        try {
+            Reader reader = new InputStreamReader(file.getInputStream());
+            mapReader = new CsvMapReader(reader, CsvPreference.STANDARD_PREFERENCE);
+
+            String[] header = mapReader.getHeader(true);
+            return header;
+
+        } catch(IOException e) {
+            e.printStackTrace();
+        } finally {
+            if( mapReader != null ) {
+                mapReader.close();
+            }
+        }
+        return null;
+    }
+
+    // TODO ms what is this for? It does ony print to stdout
     public QueryResultDto showData(Long databaseId, Long tableId) throws ImageNotSupportedException,
             DatabaseNotFoundException, TableNotFoundException, DatabaseConnectionException, DataProcessingException {
         QueryResultDto queryResult = postgresService.getAllRows(findDatabase(databaseId), findById(databaseId, tableId));
         for (Map<String, Object> m : queryResult.getResult() ) {
             for (Map.Entry<String, Object> entry : m.entrySet()) {
-                System.out.print(entry.getKey() + ": " + entry.getValue() + ", ");
+                log.debug("{}: {}", entry.getKey(), entry.getValue());
             }
         }
         return queryResult;
     }
+
+    public Table create(Long databaseId, MultipartFile file, TableCsvInformationDto tableCSVInformation) {
+        try {
+            String[] header = readHeader(file);
+            log.debug("table csv info: {}", tableCSVInformation);
+            TableCreateDto tcd = new TableCreateDto();
+            tcd.setName(tableCSVInformation.getName());
+            tcd.setDescription(tableCSVInformation.getDescription());
+            ColumnCreateDto[] cdtos = new ColumnCreateDto[header.length];
+            log.debug("header: {}", header);
+            for (int i = 0; i < header.length; i++) {
+                ColumnCreateDto c = new ColumnCreateDto();
+                c.setName(header[i]);
+                c.setType(tableCSVInformation.getColumns().get(i));
+                c.setNullAllowed(true);
+                //TODO FIX THAT not only id is primary key
+                if(header[i].equals("id")) {
+                    c.setPrimaryKey(true);
+                } else {
+                    c.setPrimaryKey(false);
+                }
+                cdtos[i] = c;
+            }
+            tcd.setColumns(cdtos);
+            Table table = create(databaseId, tcd);
+            QueryResultDto insert = insert(databaseId, table.getId(), file);
+            return table;
+        } catch (Exception e) {
+            e.printStackTrace();
+            log.error(e.getMessage());
+        }
+        return null;
+    }
+
+    public Table create(Long databaseId, TableCsvInformationDto tableCSVInformation) throws IOException {
+        Path path = Paths.get("/tmp/" + tableCSVInformation.getFileLocation());
+        String contentType = "multipart/form-data";
+        byte[] content = null;
+        try {
+            content = Files.readAllBytes(path);
+        } catch (final IOException e) {
+        }
+        MultipartFile multipartFile = new MockMultipartFile(tableCSVInformation.getFileLocation(),
+                tableCSVInformation.getFileLocation(), contentType, content);
+        Files.deleteIfExists(path);
+        return create(databaseId, multipartFile,tableCSVInformation);
+    }
+
+
 }

fda-ui/.env

@@ -2,3 +2,4 @@
 API_CONTAINER="http://localhost:9091"
 API_DATABASE="http://localhost:9092"
 API_TABLES="http://localhost:9094"
+API_ANALYSE="http://localhost:5000"

fda-ui/Dockerfile

@@ -14,6 +14,7 @@ COPY ./components ./components
 COPY ./lang ./lang
 COPY ./layouts ./layouts
 COPY ./middleware ./middleware
+COPY ./server-middleware ./server-middleware
 COPY ./pages ./pages
 COPY ./plugins ./plugins
 COPY ./store ./store

fda-ui/components/TableCreate.vue

 <template>
+  <div>
     <v-card>
       <v-card-title class="pb-0">
         Create Table
@@ -51,6 +52,7 @@
         </v-btn>
       </v-card-actions>
     </v-card>
+  </div>
 </template>
 
 <script>
@@ -98,13 +100,7 @@ export default {
       const data = {
         name: this.name,
         description: this.description,
-        columns: this.columns.map((c) => {
-          // c.nullAllowed = c.isNullAllowed
-          // c.primaryKey = c.isPrimaryKey
-          // delete c.isPrimaryKey
-          // delete c.isNullAllowed
-          return c
-        })
+        columns: this.columns
       }
       try {
         const res = await this.$axios.post(`/api/tables/api/database/${this.$route.params.db_id}/table`, data)

fda-ui/nuxt.config.js

+import path from 'path'
 import colors from 'vuetify/es5/util/colors'
 import isDocker from 'is-docker'
 
@@ -76,9 +77,14 @@ export default {
   proxy: {
     '/api/container': process.env.API_CONTAINER,
     '/api/database': process.env.API_DATABASE,
+    '/api/analyse': process.env.API_ANALYSE,
     '/api/tables': { target: process.env.API_TABLES, pathRewrite: { '^/api/tables/': '' } }
   },
 
+  serverMiddleware: [
+    { path: '/server-middleware', handler: path.resolve(__dirname, 'server-middleware/index.js') }
+  ],
+
   // Vuetify module configuration (https://go.nuxtjs.dev/config-vuetify)
   vuetify: {
     customVariables: ['~/assets/variables.scss'],

fda-ui/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "scripts": {
-    "dev": "nuxt",
+    "dev": "nuxt --port 3001",
     "docker": "nuxt > /dev/null",
     "build": "nuxt build",
     "start": "nuxt start",
@@ -22,6 +22,8 @@
     "core-js": "^3.6.5",
     "date-fns": "^2.16.1",
     "is-docker": "^2.2.1",
+    "multer": "^1.4.2",
+    "node-fetch": "^2.6.1",
     "nuxt": "^2.12.2",
     "nuxt-i18n": "^6.15.4",
     "vue-toast-notification": "^0.5.4",

fda-ui/pages/db/_db_id/tables/index.vue

@@ -5,6 +5,13 @@
     </h3>
     <TableList />
     <TableCreate />
+    <v-card class="mt-1">
+      <v-card-text>
+        <nuxt-link class="table_from_csv" :to="`/db/${$route.params.db_id}/tables/table_from_csv`">
+          Create table from CSV file
+        </nuxt-link>
+      </v-card-text>
+    </v-card>
   </div>
 </template>
 <script>
@@ -27,5 +34,8 @@ export default {
 }
 </script>
 
-<style>
+<style scoped>
+a.table_from_csv {
+  font-size: 14pt;
+}
 </style>

fda-ui/pages/db/_db_id/tables/table_from_csv.vue

+<template>
+  <div>
+    <h3 class="mb-2 mt-1">Table from CSV</h3>
+    <v-stepper v-model="step" vertical>
+      <v-stepper-step :complete="step > 1" step="1">
+        Table
+      </v-stepper-step>
+
+      <v-stepper-content class="pt-0 pb-1" step="1">
+        <v-text-field v-model="tableName" required label="Name" />
+        <v-text-field v-model="tableDesc" label="Description" />
+        <v-btn :disabled="!step1Valid" color="primary" @click="step = 2">
+          Continue
+        </v-btn>
+      </v-stepper-content>
+
+      <v-stepper-step :complete="step > 2" step="2">
+        Upload CSV file
+      </v-stepper-step>
+
+      <v-stepper-content step="2">
+        <v-row dense>
+          <v-col cols="8">
+            <v-file-input
+              v-model="file"
+              accept="text/csv"
+              show-size
+              label="CSV File" />
+          </v-col>
+          <v-col cols="4" class="mt-3">
+            <v-btn :disabled="!file" :loading="loading" @click="upload">Upload</v-btn>
+          </v-col>
+        </v-row>
+      </v-stepper-content>
+
+      <v-stepper-step :complete="step > 3" step="3">
+        Choose data type of columns
+      </v-stepper-step>
+      <v-stepper-content step="3">
+        <div v-for="(c, idx) in columns" :key="idx">
+          <v-row dense class="column pa-2 ml-1 mr-1 mb-2">
+            <v-col cols="4">
+              <v-text-field v-model="c.name" disabled required label="Name" />
+            </v-col>
+            <v-col cols="3">
+              <v-select
+                v-model="c.type"
+                :items="columnTypes"
+                item-value="value"
+                required
+                label="Data Type" />
+            </v-col>
+            <v-col cols="auto" class="pl-2">
+              <v-checkbox v-model="c.primaryKey" label="Primary Key" />
+            </v-col>
+            <v-col cols="auto" class="pl-10">
+              <v-checkbox v-model="c.nullAllowed" label="Null Allowed" />
+            </v-col>
+          </v-row>
+        </div>
+
+        <v-btn class="mt-2" color="primary" @click="createTable">
+          Continue
+        </v-btn>
+      </v-stepper-content>
+
+      <v-stepper-step
+        :complete="step > 4"
+        step="4">
+        Done
+      </v-stepper-step>
+
+      <v-stepper-content step="4">
+        Proceed to table view.
+        <div class="mt-2">
+          <v-btn :to="`/db/${$route.params.db_id}/tables/${newTableId}`" outlined>
+            <v-icon>mdi-table</v-icon>
+            View
+          </v-btn>
+        </div>
+      </v-stepper-content>
+    </v-stepper>
+  </div>
+</template>
+<script>
+export default {
+  name: 'TableFromCSV',
+  components: {
+  },
+  data () {
+    return {
+      step: 1,
+      tableName: '',
+      tableDesc: '',
+      loading: false,
+      file: null,
+      fileLocation: null,
+      columns: [],
+      columnTypes: [
+        { value: 'ENUM', text: 'ENUM' },
+        { value: 'BOOLEAN', text: 'BOOLEAN' },
+        { value: 'NUMBER', text: 'NUMBER' },
+        { value: 'BLOB', text: 'BLOB' },
+        { value: 'DATE', text: 'DATE' },
+        { value: 'STRING', text: 'STRING' },
+        { value: 'TEXT', text: 'TEXT' }
+      ],
+      newTableId: 42
+    }
+  },
+  computed: {
+    step1Valid () {
+      return this.tableName.length
+    }
+  },
+  mounted () {
+  },
+  methods: {
+    async upload () {
+      this.loading = true
+      const url = '/server-middleware/table_from_csv'
+      const data = new FormData()
+      data.append('file', this.file)
+      try {
+        const res = await this.$axios.post(url, data, {
+          headers: { 'Content-Type': 'multipart/form-data' }
+        })
+        if (res.data.success) {
+          this.columns = res.data.columns
+          this.fileLocation = res.data.file.filename
+          this.step = 3
+        } else {
+          this.$toast.error('Could not upload CSV data')
+        }
+      } catch (err) {
+        this.$toast.error('Could not upload data.')
+      }
+      this.loading = false
+    },
+    async createTable () {
+      const url = `/api/tables/api/database/${this.$route.params.db_id}/table/csv/local`
+      const data = {
+        columns: this.columns.map(c => c.type),
+        description: this.tableDesc,
+        name: this.tableName,
+        fileLocation: this.fileLocation
+      }
+      let res
+      try {
+        res = await this.$axios.post(url, data)
+        this.newTableId = res.data.id
+      } catch (err) {
+        console.log(err)
+      }
+      if (res && res.data && res.data.id) {
+        this.step = 4
+      } else {
+        this.$toast.error('Could not create table.')
+      }
+    }
+  }
+}
+</script>
+
+<style scoped>
+</style>

fda-ui/server-middleware/index.js

-export default function (req, res, next) {
-  // req is the Node.js http request object
-  console.log(req.url)
+// const bodyParser = require('body-parser')
+const app = require('express')()
+const multer = require('multer')
+const upload = multer({ dest: '/tmp' })
+const fetch = require('node-fetch')
 
-  // res is the Node.js http response object
+// TODO extend me
+const colTypeMap = {
+  Boolean: 'BOOLEAN',
+  Date: 'DATE',
+  Integer: 'NUMBER',
+  Numeric: 'NUMBER',
+  String: 'STRING',
+  Timestamp: 'DATE'
+}
+
+app.post('/table_from_csv', upload.single('file'), async (req, res) => {
+  const { file } = req
+  const { path } = file
 
-  // next is a function to call to invoke the next middleware
-  // Don't forget to call next at the end if your middleware is not an endpoint!
-  // next()
-  console.log(res)
+  // send path to analyse service
+  let analysis
+  try {
+    analysis = await fetch(`${process.env.API_ANALYSE}/datatypesbypath?filepath=${path}`)
+    analysis = await analysis.json()
+  } catch (error) {
+    return res.json({ success: false, error })
+  }
 
-  return res.json({ hi: 'foo' })
+  // map messytables / CoMi's `determine_dt` column types to ours
+  // e.g. "Integer" -> "NUMBER"
+  let entries = Object.entries(analysis.columns)
+  entries = entries.map(([k, v]) => {
+    if (colTypeMap[v]) {
+      v = colTypeMap[v]
     }
+    return {
+      name: k,
+      type: v,
+      nullAllowed: true,
+      primaryKey: false
+    }
+  })
+
+  res.json({ success: true, file, columns: entries })
+})
+
+module.exports = app

fda-ui/yarn.lock

@@ -1682,6 +1682,11 @@ anymatch@~3.1.1:
     normalize-path "^3.0.0"
     picomatch "^2.0.4"
 
+append-field@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/append-field/-/append-field-1.0.0.tgz#1e3440e915f0b1203d23748e78edd7b9b5b43e56"
+  integrity sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY=
+
 aproba@^1.1.1:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz"
@@ -2114,6 +2119,14 @@ builtin-status-codes@^3.0.0:
   resolved "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz"
   integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=
 
+busboy@^0.2.11:
+  version "0.2.14"
+  resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.2.14.tgz#6c2a622efcf47c57bbbe1e2a9c37ad36c7925453"
+  integrity sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=
+  dependencies:
+    dicer "0.2.5"
+    readable-stream "1.1.x"
+
 bytes@3.0.0:
   version "3.0.0"
   resolved "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz"
@@ -2516,7 +2529,7 @@ concat-map@0.0.1:
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz"
   integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=
 
-concat-stream@^1.5.0:
+concat-stream@^1.5.0, concat-stream@^1.5.2:
   version "1.6.2"
   resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz"
   integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==
@@ -3007,6 +3020,14 @@ detect-libc@^1.0.3:
   resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz"
   integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=
 
+dicer@0.2.5:
+  version "0.2.5"
+  resolved "https://registry.yarnpkg.com/dicer/-/dicer-0.2.5.tgz#5996c086bb33218c812c090bddc09cd12facb70f"
+  integrity sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=
+  dependencies:
+    readable-stream "1.1.x"
+    streamsearch "0.1.2"
+
 diffie-hellman@^5.0.0:
   version "5.0.3"
   resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz"
@@ -4681,6 +4702,11 @@ is-wsl@^1.1.0:
   resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-1.1.0.tgz"
   integrity sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=
 
+isarray@0.0.1:
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
+  integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=
+
 isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz"
@@ -5264,6 +5290,20 @@ ms@2.1.2:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz"
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
 
+multer@^1.4.2:
+  version "1.4.2"
+  resolved "https://registry.yarnpkg.com/multer/-/multer-1.4.2.tgz#2f1f4d12dbaeeba74cb37e623f234bf4d3d2057a"
+  integrity sha512-xY8pX7V+ybyUpbYMxtjM9KAiD9ixtg5/JkeKUTD6xilfDv0vzzOFcCp4Ljb1UU3tSOM3VTZtKo63OmzOrGi3Cg==
+  dependencies:
+    append-field "^1.0.0"
+    busboy "^0.2.11"
+    concat-stream "^1.5.2"
+    mkdirp "^0.5.1"
+    object-assign "^4.1.1"
+    on-finished "^2.3.0"
+    type-is "^1.6.4"
+    xtend "^4.0.0"
+
 multimap@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/multimap/-/multimap-1.1.0.tgz"
@@ -6725,6 +6765,16 @@ read-pkg@^5.2.0:
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
+readable-stream@1.1.x:
+  version "1.1.14"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9"
+  integrity sha1-fPTFTvZI44EwhMY23SB54WbAgdk=
+  dependencies:
+    core-util-is "~1.0.0"
+    inherits "~2.0.1"
+    isarray "0.0.1"
+    string_decoder "~0.10.x"
+
 readable-stream@^3.1.1, readable-stream@^3.6.0:
   version "3.6.0"
   resolved "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz"
@@ -7371,6 +7421,11 @@ stream-shift@^1.0.0:
   resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz"
   integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==
 
+streamsearch@0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"
+  integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=
+
 strict-uri-encode@^1.0.0:
   version "1.1.0"
   resolved "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz"
@@ -7425,6 +7480,11 @@ string_decoder@^1.0.0, string_decoder@^1.1.1:
   dependencies:
     safe-buffer "~5.2.0"
 
+string_decoder@~0.10.x:
+  version "0.10.31"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
+  integrity sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=
+
 string_decoder@~1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz"
@@ -7747,7 +7807,7 @@ type-fest@^0.8.1:
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz"
   integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==
 
-type-is@~1.6.17, type-is@~1.6.18:
+type-is@^1.6.4, type-is@~1.6.17, type-is@~1.6.18:
   version "1.6.18"
   resolved "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz"
   integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==