diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java
index fb186a6474df5cddc19fe8fed680c220d8833cfb..3672082da6ca91345b20a71a668a40f335ed76b7 100644
--- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java
+++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java
@@ -117,8 +117,16 @@ public interface QueryMapper {
.build();
}
- default PreparedStatement generateTemporaryTableSQL(Connection connection, Table table) throws QueryMalformedException {
- final StringBuilder statement = new StringBuilder("CREATE TABLE `")
+ default void importCsvQuery(Connection connection, Table table, ImportDto csv) throws SQLException {
+ final Statement statement = connection.createStatement();
+ final StringBuilder query0 = new StringBuilder("DROP TABLE IF EXISTS `")
+ .append(table.getDatabase().getInternalName())
+ .append("`.`")
+ .append(table.getInternalName())
+ .append("_temporary`;");
+ log.trace("mapped drop temporary table statement: {}", query0);
+ statement.execute(query0.toString());
+ final StringBuilder query1 = new StringBuilder("CREATE TABLE `")
.append(table.getDatabase().getInternalName())
.append("`.`")
.append(table.getInternalName())
@@ -128,33 +136,18 @@ public interface QueryMapper {
.append("`.`")
.append(table.getInternalName())
.append("`;");
- try {
- final PreparedStatement pstmt = connection.prepareStatement(statement.toString());
- log.trace("mapped create table {} to prepared statement {}", table.getName(), pstmt);
- return pstmt;
- } catch (SQLException e) {
- log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage());
- throw new QueryMalformedException("Failed to prepare statement", e);
- }
- }
+ log.trace("mapped create temporary table statement: {}", query1);
+ statement.execute(query1.toString());
+ final String query2 = pathToRawInsertQuery(table, csv);
+ log.trace("mapped import csv statement: {}", query2);
+ statement.execute(query2.toString());
+ final String query3 = generateInsertFromTemporaryTableSQL(table);
+ log.trace("mapped import table statement: {}", query3);
+ statement.execute(query3.toString());
- default PreparedStatement dropTemporaryTableSQL(Connection connection, Table table) throws QueryMalformedException {
- final StringBuilder statement = new StringBuilder("DROP TABLE IF EXISTS `")
- .append(table.getDatabase().getInternalName())
- .append("`.`")
- .append(table.getInternalName())
- .append("_temporary`;");
- try {
- final PreparedStatement pstmt = connection.prepareStatement(statement.toString());
- log.trace("mapped drop temporary table {} to prepared statement {}", table.getName(), pstmt);
- return pstmt;
- } catch (SQLException e) {
- log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage());
- throw new QueryMalformedException("Failed to prepare statement", e);
- }
}
- default PreparedStatement pathToRawInsertQuery(Connection connection, Table table, ImportDto data) throws QueryMalformedException {
+ default String pathToRawInsertQuery(Table table, ImportDto data) {
final StringBuilder statement = new StringBuilder("LOAD DATA INFILE '/tmp/")
.append(data.getLocation())
.append("' INTO TABLE `")
@@ -202,14 +195,7 @@ public interface QueryMapper {
statement.append(")")
.append(set.length() != 0 ? (" SET " + set) : "")
.append(";");
- try {
- final PreparedStatement pstmt = connection.prepareStatement(statement.toString());
- log.trace("mapped drop temporary table {} to prepared statement {}", table.getName(), pstmt);
- return pstmt;
- } catch (SQLException e) {
- log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage());
- throw new QueryMalformedException("Failed to prepare statement", e);
- }
+ return statement.toString();
}
default void columnToBoolSet(ImportDto data, TableColumn column, StringBuilder set) {
@@ -869,8 +855,7 @@ public interface QueryMapper {
return "`" + item.substring(idx + 1) + "`";
}
- default PreparedStatement generateInsertFromTemporaryTableSQL(Connection connection, Table table)
- throws QueryMalformedException {
+ default String generateInsertFromTemporaryTableSQL(Table table) {
final StringBuilder statement = new StringBuilder("INSERT INTO `")
.append(table.getDatabase().getInternalName())
.append("`.`")
@@ -898,14 +883,7 @@ public interface QueryMapper {
.append("`),");
statement.deleteCharAt(statement.length() - 1);
statement.append(";");
- try {
- final PreparedStatement pstmt = connection.prepareStatement(statement.toString());
- log.trace("mapped generate insert from temporary table {} to prepared statement {}", statement, pstmt);
- return pstmt;
- } catch (SQLException e) {
- log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage());
- throw new QueryMalformedException("Failed to prepare statement", e);
- }
+ return statement.toString();
}
default void prepareStatementWithColumnTypeObject(PreparedStatement ps, TableColumnType columnType, int idx, Object value) throws SQLException {
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
index e2cf9491d49448784eaa11fe51a1abf2e5aae173..ca2aca3e1d9b6e83e01511d95cf5403e18b48163 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
@@ -401,40 +401,17 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
/* find */
final Database database = databaseService.find(databaseId);
final Table table = tableService.find(databaseId, tableId);
- /* preparing the statements */
- final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(),
- database.getContainer(), database);
- /* Create a temporary table, insert there, transfer with update on duplicate key and lastly drops the temporary table */
- try {
- final Connection connection = dataSource.getConnection();
- queryMapper.dropTemporaryTableSQL(connection, table)
- .executeUpdate();
- } catch (SQLException e) {
- log.error("Failed to drop temporary table: {}", e.getMessage());
- throw new TableMalformedException("Failed to drop temporary table", e);
- }
- try {
- final Connection connection = dataSource.getConnection();
- queryMapper.generateTemporaryTableSQL(connection, table)
- .executeUpdate();
- } catch (SQLException e) {
- log.error("Failed to create temporary table: {}", e.getMessage());
- dataSource.close();
- throw new TableMalformedException("Failed to create temporary table", e);
- }
/* import .csv from blob storage to sidecar */
dataDbSidecarGateway.importFile(database.getContainer().getSidecarHost(), database.getContainer().getSidecarPort(), data.getLocation());
/* import .csv from sidecar to database */
+ final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(),
+ database.getContainer(), database);
try {
final Connection connection = dataSource.getConnection();
- queryMapper.pathToRawInsertQuery(connection, table, data)
- .executeUpdate();
- queryMapper.generateInsertFromTemporaryTableSQL(connection, table)
- .executeUpdate();
+ queryMapper.importCsvQuery(connection, table, data);
} catch (SQLException e) {
- log.error("Failed to insert temporary table: {}", e.getMessage());
- dataSource.close();
- throw new TableMalformedException("Failed to insert temporary table", e);
+ log.error("Failed to import .csv: {}", e.getMessage());
+ throw new TableMalformedException("Failed to import .csv", e);
} finally {
dataSource.close();
}
diff --git a/dbrepo-ui/components/dialogs/EditTuple.vue b/dbrepo-ui/components/dialogs/EditTuple.vue
index 9547f44d6893510c9b52cadae990de3fa74e9f5e..d3f240100c500b27699675f47646857c8ed6ef71 100644
--- a/dbrepo-ui/components/dialogs/EditTuple.vue
+++ b/dbrepo-ui/components/dialogs/EditTuple.vue
@@ -240,13 +240,11 @@ export default {
return ['date', 'datetime', 'timestamp', 'time', 'year'].includes(column.column_type)
},
rules (column) {
- if (column.auto_generated) {
+ if (column.auto_generated || column.is_null_allowed) {
return []
}
const rules = []
- if (!column.is_null_allowed) {
- rules.push(v => !!v || 'Required')
- }
+ rules.push(v => !!v || 'Required')
if (column.column_type === 'char') {
rules.push(v => !(!v || v.length !== column.size) || `Must be exactly ${column.size} character${column.size !== 1 ? 's' : ''}`)
}
diff --git a/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue b/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue
index 5fa9cc6a387e3ae89cd8e9864d6d1749f215a25c..1f60efcb8d816ec70ab82bb2325862e15f24a328 100644
--- a/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue
+++ b/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue
@@ -24,6 +24,7 @@
item-text="key"
item-value="value"
required
+ clearable
hint="Character separating the values"
label="Separator *" />
</v-col>
@@ -35,6 +36,7 @@
:rules="[v => isNonNegativeInteger(v) || $t('Greater or equal to zero')]"
type="number"
required
+ clearable
hint="Skip n lines from the top. These may include comments or the header of column names."
label="Number of lines to skip *"
placeholder="e.g. 0" />
@@ -47,6 +49,7 @@
:items="quotes"
item-text="key"
item-value="value"
+ clearable
hint="Character quoting the values"
label="Value quotes" />
</v-col>
@@ -57,6 +60,7 @@
v-model="tableImport.null_element"
hint="Representation of 'no value present'"
placeholder="e.g. NA"
+ clearable
label="NULL Element" />
</v-col>
</v-row>
@@ -65,6 +69,7 @@
<v-text-field
v-model="tableImport.true_element"
label="True Element"
+ clearable
hint="Representation of boolean 'true'"
placeholder="e.g. 1, true, YES" />
</v-col>
@@ -74,6 +79,7 @@
<v-text-field
v-model="tableImport.false_element"
label="False Element"
+ clearable
hint="Representation of boolean 'false'"
placeholder="e.g. 0, false, NO" />
</v-col>
@@ -85,6 +91,7 @@
accept=".csv,.tsv"
hint="max. 2GB file size"
persistent-hint
+ clearable
:show-size="1000"
counter
label="CSV/TSV File" />
diff --git a/dbrepo-ui/pages/database/_database_id/table/import.vue b/dbrepo-ui/pages/database/_database_id/table/import.vue
index 789259e4fb5fdd01580f9e4525725d6fc19cdb52..a64bcc54f605e3fea0fbec97d47eee6b41451241 100644
--- a/dbrepo-ui/pages/database/_database_id/table/import.vue
+++ b/dbrepo-ui/pages/database/_database_id/table/import.vue
@@ -57,6 +57,7 @@
item-text="key"
item-value="value"
required
+ clearable
hint="Character separating the values"
label="Separator *" />
</v-col>
@@ -69,6 +70,7 @@
v => isNonNegativeInteger(v) || $t('Greater or equal to zero')]"
type="number"
required
+ clearable
hint="Skip n lines from the top. These may include comments or the header of column names."
label="Number of lines to skip *"
placeholder="e.g. 0" />
@@ -81,6 +83,7 @@
:items="quotes"
item-text="key"
item-value="value"
+ clearable
hint="Character quoting the values"
label="Value quotes" />
</v-col>
@@ -91,6 +94,7 @@
v-model="tableImport.null_element"
hint="Representation of 'no value present'"
placeholder="e.g. NA"
+ clearable
label="NULL Element" />
</v-col>
</v-row>
@@ -99,6 +103,7 @@
<v-text-field
v-model="tableImport.true_element"
label="True Element"
+ clearable
hint="Representation of boolean 'true'"
placeholder="e.g. 1, true, YES" />
</v-col>
@@ -108,6 +113,7 @@
<v-text-field
v-model="tableImport.false_element"
label="False Element"
+ clearable
hint="Representation of boolean 'false'"
placeholder="e.g. 0, false, NO" />
</v-col>
@@ -329,6 +335,7 @@ export default {
UploadService.upload(this.fileModel)
.then((metadata) => {
console.debug('uploaded file', metadata)
+ this.loading = false
resolve(metadata)
})
.catch((error) => {