Skip to content
Snippets Groups Projects
Unverified Commit 45a099aa authored by Martin Weise's avatar Martin Weise
Browse files

Updated import statement to one tx, updated insert data dialog

parent d8e820cc
No related branches found
No related tags found
4 merge requests!231CI: Remove build for log-service,!228Better error message handling in the frontend,!223Release of version 1.4.0,!213Resolve "Some bugs regarding data"
Show whitespace changes
Inline Side-by-side
Showing
with 43 additions and 76 deletions
dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java
+ 22
− 44
View file @ 45a099aa
......@@ -117,8 +117,16 @@ public interface QueryMapper {
.build();
}
default PreparedStatement generateTemporaryTableSQL(Connection connection, Table table) throws QueryMalformedException {
final StringBuilder statement = new StringBuilder("CREATE TABLE `")
default void importCsvQuery(Connection connection, Table table, ImportDto csv) throws SQLException {
final Statement statement = connection.createStatement();
final StringBuilder query0 = new StringBuilder("DROP TABLE IF EXISTS `")
.append(table.getDatabase().getInternalName())
.append("`.`")
.append(table.getInternalName())
.append("_temporary`;");
log.trace("mapped drop temporary table statement: {}", query0);
statement.execute(query0.toString());
final StringBuilder query1 = new StringBuilder("CREATE TABLE `")
.append(table.getDatabase().getInternalName())
.append("`.`")
.append(table.getInternalName())
......@@ -128,33 +136,18 @@ public interface QueryMapper {
.append("`.`")
.append(table.getInternalName())
.append("`;");
try {
final PreparedStatement pstmt = connection.prepareStatement(statement.toString());
log.trace("mapped create table {} to prepared statement {}", table.getName(), pstmt);
return pstmt;
} catch (SQLException e) {
log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage());
throw new QueryMalformedException("Failed to prepare statement", e);
}
}
log.trace("mapped create temporary table statement: {}", query1);
statement.execute(query1.toString());
final String query2 = pathToRawInsertQuery(table, csv);
log.trace("mapped import csv statement: {}", query2);
statement.execute(query2.toString());
final String query3 = generateInsertFromTemporaryTableSQL(table);
log.trace("mapped import table statement: {}", query3);
statement.execute(query3.toString());
default PreparedStatement dropTemporaryTableSQL(Connection connection, Table table) throws QueryMalformedException {
final StringBuilder statement = new StringBuilder("DROP TABLE IF EXISTS `")
.append(table.getDatabase().getInternalName())
.append("`.`")
.append(table.getInternalName())
.append("_temporary`;");
try {
final PreparedStatement pstmt = connection.prepareStatement(statement.toString());
log.trace("mapped drop temporary table {} to prepared statement {}", table.getName(), pstmt);
return pstmt;
} catch (SQLException e) {
log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage());
throw new QueryMalformedException("Failed to prepare statement", e);
}
}
default PreparedStatement pathToRawInsertQuery(Connection connection, Table table, ImportDto data) throws QueryMalformedException {
default String pathToRawInsertQuery(Table table, ImportDto data) {
final StringBuilder statement = new StringBuilder("LOAD DATA INFILE '/tmp/")
.append(data.getLocation())
.append("' INTO TABLE `")
......@@ -202,14 +195,7 @@ public interface QueryMapper {
statement.append(")")
.append(set.length() != 0 ? (" SET " + set) : "")
.append(";");
try {
final PreparedStatement pstmt = connection.prepareStatement(statement.toString());
log.trace("mapped drop temporary table {} to prepared statement {}", table.getName(), pstmt);
return pstmt;
} catch (SQLException e) {
log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage());
throw new QueryMalformedException("Failed to prepare statement", e);
}
return statement.toString();
}
default void columnToBoolSet(ImportDto data, TableColumn column, StringBuilder set) {
......@@ -869,8 +855,7 @@ public interface QueryMapper {
return "`" + item.substring(idx + 1) + "`";
}
default PreparedStatement generateInsertFromTemporaryTableSQL(Connection connection, Table table)
throws QueryMalformedException {
default String generateInsertFromTemporaryTableSQL(Table table) {
final StringBuilder statement = new StringBuilder("INSERT INTO `")
.append(table.getDatabase().getInternalName())
.append("`.`")
......@@ -898,14 +883,7 @@ public interface QueryMapper {
.append("`),");
statement.deleteCharAt(statement.length() - 1);
statement.append(";");
try {
final PreparedStatement pstmt = connection.prepareStatement(statement.toString());
log.trace("mapped generate insert from temporary table {} to prepared statement {}", statement, pstmt);
return pstmt;
} catch (SQLException e) {
log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage());
throw new QueryMalformedException("Failed to prepare statement", e);
}
return statement.toString();
}
default void prepareStatementWithColumnTypeObject(PreparedStatement ps, TableColumnType columnType, int idx, Object value) throws SQLException {
......
......
dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
+ 5
− 28
View file @ 45a099aa
......@@ -401,40 +401,17 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
/* find */
final Database database = databaseService.find(databaseId);
final Table table = tableService.find(databaseId, tableId);
/* preparing the statements */
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(),
database.getContainer(), database);
/* Create a temporary table, insert there, transfer with update on duplicate key and lastly drops the temporary table */
try {
final Connection connection = dataSource.getConnection();
queryMapper.dropTemporaryTableSQL(connection, table)
.executeUpdate();
} catch (SQLException e) {
log.error("Failed to drop temporary table: {}", e.getMessage());
throw new TableMalformedException("Failed to drop temporary table", e);
}
try {
final Connection connection = dataSource.getConnection();
queryMapper.generateTemporaryTableSQL(connection, table)
.executeUpdate();
} catch (SQLException e) {
log.error("Failed to create temporary table: {}", e.getMessage());
dataSource.close();
throw new TableMalformedException("Failed to create temporary table", e);
}
/* import .csv from blob storage to sidecar */
dataDbSidecarGateway.importFile(database.getContainer().getSidecarHost(), database.getContainer().getSidecarPort(), data.getLocation());
/* import .csv from sidecar to database */
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(),
database.getContainer(), database);
try {
final Connection connection = dataSource.getConnection();
queryMapper.pathToRawInsertQuery(connection, table, data)
.executeUpdate();
queryMapper.generateInsertFromTemporaryTableSQL(connection, table)
.executeUpdate();
queryMapper.importCsvQuery(connection, table, data);
} catch (SQLException e) {
log.error("Failed to insert temporary table: {}", e.getMessage());
dataSource.close();
throw new TableMalformedException("Failed to insert temporary table", e);
log.error("Failed to import .csv: {}", e.getMessage());
throw new TableMalformedException("Failed to import .csv", e);
} finally {
dataSource.close();
}
......
......
dbrepo-ui/components/dialogs/EditTuple.vue
+ 2
− 4
View file @ 45a099aa
......@@ -240,13 +240,11 @@ export default {
return ['date', 'datetime', 'timestamp', 'time', 'year'].includes(column.column_type)
},
rules (column) {
if (column.auto_generated) {
if (column.auto_generated || column.is_null_allowed) {
return []
}
const rules = []
if (!column.is_null_allowed) {
rules.push(v => !!v || 'Required')
}
if (column.column_type === 'char') {
rules.push(v => !(!v || v.length !== column.size) || `Must be exactly ${column.size} character${column.size !== 1 ? 's' : ''}`)
}
......
......
dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue
+ 7
− 0
View file @ 45a099aa
......@@ -24,6 +24,7 @@
item-text="key"
item-value="value"
required
clearable
hint="Character separating the values"
label="Separator *" />
</v-col>
......@@ -35,6 +36,7 @@
:rules="[v => isNonNegativeInteger(v) || $t('Greater or equal to zero')]"
type="number"
required
clearable
hint="Skip n lines from the top. These may include comments or the header of column names."
label="Number of lines to skip *"
placeholder="e.g. 0" />
......@@ -47,6 +49,7 @@
:items="quotes"
item-text="key"
item-value="value"
clearable
hint="Character quoting the values"
label="Value quotes" />
</v-col>
......@@ -57,6 +60,7 @@
v-model="tableImport.null_element"
hint="Representation of 'no value present'"
placeholder="e.g. NA"
clearable
label="NULL Element" />
</v-col>
</v-row>
......@@ -65,6 +69,7 @@
<v-text-field
v-model="tableImport.true_element"
label="True Element"
clearable
hint="Representation of boolean 'true'"
placeholder="e.g. 1, true, YES" />
</v-col>
......@@ -74,6 +79,7 @@
<v-text-field
v-model="tableImport.false_element"
label="False Element"
clearable
hint="Representation of boolean 'false'"
placeholder="e.g. 0, false, NO" />
</v-col>
......@@ -85,6 +91,7 @@
accept=".csv,.tsv"
hint="max. 2GB file size"
persistent-hint
clearable
:show-size="1000"
counter
label="CSV/TSV File" />
......
......
dbrepo-ui/pages/database/_database_id/table/import.vue
+ 7
− 0
View file @ 45a099aa
......@@ -57,6 +57,7 @@
item-text="key"
item-value="value"
required
clearable
hint="Character separating the values"
label="Separator *" />
</v-col>
......@@ -69,6 +70,7 @@
v => isNonNegativeInteger(v) || $t('Greater or equal to zero')]"
type="number"
required
clearable
hint="Skip n lines from the top. These may include comments or the header of column names."
label="Number of lines to skip *"
placeholder="e.g. 0" />
......@@ -81,6 +83,7 @@
:items="quotes"
item-text="key"
item-value="value"
clearable
hint="Character quoting the values"
label="Value quotes" />
</v-col>
......@@ -91,6 +94,7 @@
v-model="tableImport.null_element"
hint="Representation of 'no value present'"
placeholder="e.g. NA"
clearable
label="NULL Element" />
</v-col>
</v-row>
......@@ -99,6 +103,7 @@
<v-text-field
v-model="tableImport.true_element"
label="True Element"
clearable
hint="Representation of boolean 'true'"
placeholder="e.g. 1, true, YES" />
</v-col>
......@@ -108,6 +113,7 @@
<v-text-field
v-model="tableImport.false_element"
label="False Element"
clearable
hint="Representation of boolean 'false'"
placeholder="e.g. 0, false, NO" />
</v-col>
......@@ -329,6 +335,7 @@ export default {
UploadService.upload(this.fileModel)
.then((metadata) => {
console.debug('uploaded file', metadata)
this.loading = false
resolve(metadata)
})
.catch((error) => {
......
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment