From c9fd3c50e4aa3c6b7c002ccfe1bdf0ef2077f0eb Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Thu, 23 Jan 2025 22:31:30 +0100
Subject: [PATCH] Fixed some UI bugs and paginating for Spark
Signed-off-by: Martin Weise <martin.weise@tuwien.ac.at>
---
dbrepo-data-service/Dockerfile | 4 +-
.../at/tuwien/endpoints/RestEndpoint.java | 1 +
.../impl/MetadataServiceGatewayImpl.java | 1 +
.../java/at/tuwien/mapper/MariaDbMapper.java | 47 +++++++++----------
.../at/tuwien/service/impl/DataConnector.java | 3 +-
.../service/impl/TableServiceMariaDbImpl.java | 14 ++++--
dbrepo-metadata-service/Dockerfile | 4 +-
dbrepo-ui/components/subset/Results.vue | 18 +++++--
dbrepo-ui/components/subset/SubsetToolbar.vue | 12 +++--
dbrepo-ui/composables/table-service.ts | 26 +++++-----
dbrepo-ui/nuxt.config.ts | 4 +-
.../[database_id]/subset/[subset_id]/data.vue | 8 ++--
.../[database_id]/table/[table_id]/data.vue | 2 +-
dbrepo-ui/stores/cache.js | 8 ++++
14 files changed, 88 insertions(+), 64 deletions(-)
diff --git a/dbrepo-data-service/Dockerfile b/dbrepo-data-service/Dockerfile
index f4e2be5b96..4b45e94290 100644
--- a/dbrepo-data-service/Dockerfile
+++ b/dbrepo-data-service/Dockerfile
@@ -8,7 +8,7 @@ LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
COPY ./pom.xml ./
-RUN mvn -fn -B dependency:go-offline
+RUN mvn -fn -B -q dependency:go-offline
COPY --from=dependency /root/.m2/repository/at/tuwien /root/.m2/repository/at/tuwien
@@ -18,7 +18,7 @@ COPY ./rest-service ./rest-service
COPY ./services ./services
# Make sure it compiles
-RUN mvn clean package -DskipTests
+RUN mvn -fn -B -q clean package -DskipTests
###### THIRD STAGE ######
FROM amazoncorretto:17-alpine3.19 AS runtime
diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/RestEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/RestEndpoint.java
index 333e0c8398..45cea4371c 100644
--- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/RestEndpoint.java
+++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/RestEndpoint.java
@@ -45,6 +45,7 @@ public abstract class RestEndpoint {
return UUID.fromString(user.getId());
}
+ /* FIXME: Heap may run OOM */
public List<Map<String, Object>> transform(Dataset<Row> dataset) {
return dataset.collectAsList()
.stream()
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java
index 4dcfaf13a2..650652d62f 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java
@@ -115,6 +115,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway {
database.setJdbcMethod(response.getHeaders().get("X-Type").get(0));
database.setUsername(response.getHeaders().get("X-Username").get(0));
database.setPassword(response.getHeaders().get("X-Password").get(0));
+ database.setDatabase(database.getInternalName());
database.setHost(response.getHeaders().get("X-Host").get(0));
database.setPort(Integer.parseInt(response.getHeaders().get("X-Port").get(0)));
database.setLastRetrieved(Instant.now());
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
index 7f5b63b21c..d3af548164 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
@@ -231,12 +231,6 @@ public interface MariaDbMapper {
return statement.toString();
}
- default String tableCreateDtoToSequenceName(at.tuwien.api.database.table.internal.TableCreateDto data) {
- final String name = "seq_" + nameToInternalName(data.getName()) + "_id";
- log.trace("mapped table name {} to sequence name {}", data.getName(), name);
- return name;
- }
-
/**
* Maps the desired data type to a MySQL string with the default MySQL 8 values for each
*
@@ -731,27 +725,30 @@ public interface MariaDbMapper {
}
}
- default String selectRawSelectQuery(String query, Instant timestamp, Long page, Long size) {
- query = query.toLowerCase(Locale.ROOT)
- .trim();
- if (query.matches(";$")) {
- /* remove last semicolon */
- query = query.substring(0, query.length() - 1);
- }
+ default String defaultRawSelectQuery(String databaseName, String tableOrViewName, Instant timestamp, Long page,
+ Long size) {
/* query check (this is enforced by the db also) */
- final StringBuilder statement = new StringBuilder("SELECT * FROM (")
- .append(query)
- .append(") FOR SYSTEM_TIME AS OF TIMESTAMP '")
- .append(mariaDbFormatter.format(timestamp))
- .append("' as tbl");
+ final StringBuilder statement = new StringBuilder("SELECT * FROM (SELECT * FROM `")
+ .append(databaseName)
+ .append("`.`")
+ .append(tableOrViewName)
+ .append("`");
+ if (timestamp != null) {
+ statement.append(" FOR SYSTEM_TIME AS OF TIMESTAMP '")
+ .append(mariaDbFormatter.format(timestamp))
+ .append("'");
+ }
+ statement.append(" as tbl");
/* pagination */
- log.trace("pagination size/limit of {}", size);
- statement.append(" LIMIT ")
- .append(size);
- log.trace("pagination page/offset of {}", page);
- statement.append(" OFFSET ")
- .append(page * size);
- statement.append(";");
+ if (size != null && page != null) {
+ log.trace("pagination size/limit of {}", size);
+ statement.append(" LIMIT ")
+ .append(size);
+ log.trace("pagination page/offset of {}", page);
+ statement.append(" OFFSET ")
+ .append(page * size);
+ }
+ statement.append(") as tbl2");
log.trace("mapped select query: {}", statement);
return statement.toString();
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java
index 1044869ad8..3a54b399d4 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java
@@ -10,7 +10,6 @@ import org.springframework.stereotype.Service;
public abstract class DataConnector<T extends CacheableDto> {
public ComboPooledDataSource getDataSource(T entity) {
- final long start = System.currentTimeMillis();
final ComboPooledDataSource dataSource = new ComboPooledDataSource();
dataSource.setJdbcUrl(getJdbcUrl(entity.getJdbcMethod(), entity.getHost(), entity.getPort(),
entity.getDatabase()));
@@ -25,7 +24,6 @@ public abstract class DataConnector<T extends CacheableDto> {
}
public ComboPooledDataSource getDataSource(T entity, String databaseName) {
- final long start = System.currentTimeMillis();
final ComboPooledDataSource dataSource = new ComboPooledDataSource();
dataSource.setJdbcUrl(getJdbcUrl(entity.getJdbcMethod(), entity.getHost(), entity.getPort(), databaseName));
dataSource.setUser(entity.getUsername());
@@ -60,6 +58,7 @@ public abstract class DataConnector<T extends CacheableDto> {
stringBuilder.append("/")
.append(databaseName);
}
+ log.trace("mapped jdbc url: {}", stringBuilder);
return stringBuilder.toString();
}
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
index 68fb59cb5e..482f874624 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java
@@ -373,12 +373,16 @@ public class TableServiceMariaDbImpl extends DataConnector<TableDto> implements
Long page, Long size, SortTypeDto sortDirection, String sortColumn)
throws QueryMalformedException, TableNotFoundException {
try {
- final Properties properties = new Properties();
- properties.setProperty("user", database.getUsername());
- properties.setProperty("password", database.getPassword());
return sparkSession.read()
- .jdbc(getSparkUrl(database.getJdbcMethod(), database.getHost(), database.getPort(),
- database.getInternalName()), tableOrView, properties);
+ .format("jdbc")
+ .option("user", database.getUsername())
+ .option("password", database.getPassword())
+ .option("url", getSparkUrl(database.getJdbcMethod(), database.getHost(), database.getPort(),
+ database.getInternalName()))
+ .option("query", mariaDbMapper.defaultRawSelectQuery(database.getInternalName(), tableOrView,
+ timestamp, page, size))
+ .load();
+
} catch (Exception e) {
if (e instanceof ExtendedAnalysisException exception) {
if (exception.getSimpleMessage().contains("TABLE_OR_VIEW_NOT_FOUND")) {
diff --git a/dbrepo-metadata-service/Dockerfile b/dbrepo-metadata-service/Dockerfile
index 843c334a9a..ddc20cb420 100644
--- a/dbrepo-metadata-service/Dockerfile
+++ b/dbrepo-metadata-service/Dockerfile
@@ -12,7 +12,7 @@ COPY ./rest-service/pom.xml ./rest-service/
COPY ./services/pom.xml ./services/
COPY ./test/pom.xml ./test/
-RUN mvn verify -B -fn
+RUN mvn -fn -B dependency:go-offline
COPY ./api ./api
COPY ./entities ./entities
@@ -24,7 +24,7 @@ COPY ./services ./services
COPY ./test ./test
# Make sure it compiles
-RUN mvn clean install -DskipTests
+RUN mvn -fn -B clean install -DskipTests
###### SECOND STAGE ######
FROM amazoncorretto:17-alpine3.19 AS runtime
diff --git a/dbrepo-ui/components/subset/Results.vue b/dbrepo-ui/components/subset/Results.vue
index e558186daf..3948667518 100644
--- a/dbrepo-ui/components/subset/Results.vue
+++ b/dbrepo-ui/components/subset/Results.vue
@@ -110,9 +110,13 @@ export default {
this.id = id
this.loadingExecute = false
})
- .catch(({code}) => {
+ .catch(({code, message}) => {
this.loadingExecute = false
const toast = useToastInstance()
+ if (message) {
+ toast.error(message)
+ return
+ }
if (typeof code !== 'string') {
return
}
@@ -129,9 +133,13 @@ export default {
this.id = id
this.loadingExecute = false
})
- .catch(({code}) => {
+ .catch(({code, message}) => {
this.loadingExecute = false
const toast = useToastInstance()
+ if (message) {
+ toast.error(message)
+ return
+ }
if (typeof code !== 'string') {
return
}
@@ -148,9 +156,13 @@ export default {
this.id = id
this.loadingExecute = false
})
- .catch(({code}) => {
+ .catch(({code, message}) => {
this.loadingExecute = false
const toast = useToastInstance()
+ if (message) {
+ toast.error(message)
+ return
+ }
if (typeof code !== 'string') {
return
}
diff --git a/dbrepo-ui/components/subset/SubsetToolbar.vue b/dbrepo-ui/components/subset/SubsetToolbar.vue
index 874e691c5f..d5f45e48e3 100644
--- a/dbrepo-ui/components/subset/SubsetToolbar.vue
+++ b/dbrepo-ui/components/subset/SubsetToolbar.vue
@@ -35,7 +35,6 @@
variant="flat"
class="mr-2"
:prepend-icon="$vuetify.display.lgAndUp ? 'mdi-content-save-outline' : null"
- :disabled="!executionUTC"
:to="`/database/${$route.params.database_id}/subset/${$route.params.subset_id}/persist`">
{{ ($vuetify.display.lgAndUp ? $t('toolbars.subset.pid.xl') + ' ' : '') + $t('toolbars.subset.pid.permanent') }}
</v-btn>
@@ -177,8 +176,9 @@ export default {
this.loadingSave = true
const queryService = useQueryService()
queryService.update(this.$route.params.database_id, this.$route.params.subset_id, { persist: true })
- .then((subset) => {
- this.subset = subset
+ .then(() => {
+ const cacheStore = useCacheStore()
+ cacheStore.reloadSubset()
this.loadingSave = false
})
.catch(() => {
@@ -192,8 +192,10 @@ export default {
this.loadingSave = true
const queryService = useQueryService()
queryService.update(this.$route.params.database_id, this.$route.params.subset_id, { persist: false })
- .then((subset) => {
- this.subset = subset
+ .then(() => {
+ const cacheStore = useCacheStore()
+ cacheStore.reloadSubset()
+ this.loadingSave = false
})
.catch(() => {
this.loadingSave = false
diff --git a/dbrepo-ui/composables/table-service.ts b/dbrepo-ui/composables/table-service.ts
index ca757c7451..45268d6295 100644
--- a/dbrepo-ui/composables/table-service.ts
+++ b/dbrepo-ui/composables/table-service.ts
@@ -9,7 +9,7 @@ export const useTableService = (): any => {
return new Promise<TableBriefDto>((resolve, reject) => {
axios.get<TableBriefDto>(`/api/database/${databaseId}/table`)
.then((response) => {
- console.info('Found tables(s)')
+ console.info(`Found ${response.data.length} tables(s)`)
resolve(response.data)
})
.catch((error) => {
@@ -25,7 +25,7 @@ export const useTableService = (): any => {
return new Promise<TableDto>((resolve, reject) => {
axios.get<TableDto>(`/api/database/${databaseId}/table/${tableId}`)
.then((response) => {
- console.info('Found table with id', tableId, 'in database with id', databaseId);
+ console.info('Found table');
resolve(response.data)
})
.catch((error) => {
@@ -41,7 +41,7 @@ export const useTableService = (): any => {
return new Promise<ColumnDto>((resolve, reject) => {
axios.put<ColumnDto>(`/api/database/${databaseId}/table/${tableId}/column/${columnId}`, data)
.then((response) => {
- console.info('Updated column with id', columnId, 'table with id', tableId, 'in database with id', databaseId);
+ console.info('Updated column');
resolve(response.data)
})
.catch((error) => {
@@ -57,7 +57,7 @@ export const useTableService = (): any => {
return new Promise<TableDto>((resolve, reject) => {
axios.put<TableDto>(`/api/database/${databaseId}/table/${tableId}`, data)
.then((response) => {
- console.info('Updated table with id', tableId, 'in database with id', databaseId);
+ console.info('Updated table');
resolve(response.data)
})
.catch((error) => {
@@ -73,7 +73,7 @@ export const useTableService = (): any => {
return new Promise<ImportDto>((resolve, reject) => {
axios.post<ImportDto>(`/api/database/${databaseId}/table/${tableId}/data/import`, data)
.then((response) => {
- console.info('Imported csv to table with id', tableId, 'in database with id', databaseId)
+ console.info('Imported csv to table')
resolve(response.data)
})
.catch((error) => {
@@ -89,7 +89,7 @@ export const useTableService = (): any => {
return new Promise<QueryResultDto>((resolve, reject) => {
axios.get<QueryResultDto>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, page, size) })
.then((response) => {
- console.info('Got data for table with id', tableId, 'in database with id', databaseId)
+ console.info('Got data for table')
const result: QueryResultDto = {
id: tableId,
headers: response.headers['x-headers'] ? response.headers['x-headers'].split(',') : [],
@@ -111,7 +111,7 @@ export const useTableService = (): any => {
axios.head<void>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, null, null) })
.then((response: AxiosResponse<void>) => {
const count: number = Number(response.headers['x-count'])
- console.info('Found' + count + 'in table with id', tableId, 'in database with id', databaseId)
+ console.info(`Found ${count} tuple(s)`)
resolve(count)
})
.catch((error) => {
@@ -134,7 +134,7 @@ export const useTableService = (): any => {
return new Promise<QueryResultDto>((resolve, reject) => {
axios.get<QueryResultDto>(`/api/database/${databaseId}/table/${tableId}/export`, config)
.then((response) => {
- console.info('Exported data for table with id', tableId, 'in database with id', databaseId)
+ console.info('Exported data for table')
resolve(response.data)
})
.catch((error) => {
@@ -150,7 +150,7 @@ export const useTableService = (): any => {
return new Promise<TableDto>((resolve, reject) => {
axios.post<TableDto>(`/api/database/${databaseId}/table`, data)
.then((response) => {
- console.info('Created table in database with id', databaseId)
+ console.info('Created table')
resolve(response.data)
})
.catch((error: AxiosError) => {
@@ -166,7 +166,7 @@ export const useTableService = (): any => {
return new Promise<void>((resolve, reject) => {
axios.delete<void>(`/api/database/${databaseId}/table/${tableId}`)
.then((response) => {
- console.info('Deleted table with id', tableId, 'in database with id', databaseId)
+ console.info('Deleted table')
resolve(response.data)
})
.catch((error) => {
@@ -182,7 +182,7 @@ export const useTableService = (): any => {
return new Promise<void>((resolve, reject) => {
axios.delete<void>(`/api/database/${databaseId}/table/${tableId}`, {data})
.then((response) => {
- console.info('Deleted tuple(s) in table with id', tableId, 'in database with id', databaseId)
+ console.info(`Deleted tuple(s)`)
resolve(response.data)
})
.catch((error) => {
@@ -198,7 +198,7 @@ export const useTableService = (): any => {
return new Promise<TableHistoryDto[]>((resolve, reject) => {
axios.get<TableHistoryDto[]>(`/api/database/${databaseId}/table/${tableId}/history`)
.then((response) => {
- console.info('Loaded history of table with id', tableId, 'in database with id', databaseId)
+ console.info('Loaded history of table')
resolve(response.data)
})
.catch((error) => {
@@ -214,7 +214,7 @@ export const useTableService = (): any => {
return new Promise<TableColumnEntityDto[]>((resolve, reject) => {
axios.get<TableColumnEntityDto[]>(`/api/database/${databaseId}/table/${tableId}/column/${columnId}/suggest`)
.then((response) => {
- console.info('Suggested semantic entities for table column with id', columnId, 'of table with id', tableId, 'of database with id', databaseId)
+ console.info('Suggested semantic entities')
resolve(response.data)
})
.catch((error) => {
diff --git a/dbrepo-ui/nuxt.config.ts b/dbrepo-ui/nuxt.config.ts
index 70084f3cca..4b1833d816 100644
--- a/dbrepo-ui/nuxt.config.ts
+++ b/dbrepo-ui/nuxt.config.ts
@@ -75,8 +75,8 @@ export default defineNuxtConfig({
}
},
api: {
- client: 'https://dbrepo1.ec.tuwien.ac.at',
- server: 'https://dbrepo1.ec.tuwien.ac.at',
+ client: 'http://localhost',
+ server: 'http://gateway-service',
},
upload: {
client: 'http://localhost/api/upload/files',
diff --git a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue
index f96b6e4175..e9719cd0ef 100644
--- a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue
+++ b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue
@@ -119,7 +119,7 @@ export default {
},
},
mounted () {
- this.loadResult()
+ this.loadSubset()
},
methods: {
loadSubset () {
@@ -128,7 +128,9 @@ export default {
queryService.findOne(this.$route.params.database_id, this.$route.params.subset_id)
.then((subset) => {
this.subset = subset
- this.loadResult()
+ this.$refs.queryResults.reExecute(subset.id)
+ this.$refs.queryResults.reExecuteCount(subset.id)
+ this.loadingSubset = false
})
.catch(() => {
this.loadingSubset = false
@@ -139,8 +141,6 @@ export default {
},
loadResult () {
if (this.subset) {
- this.$refs.queryResults.reExecute(this.subset.id)
- this.$refs.queryResults.reExecuteCount(this.subset.id)
}
},
download () {
diff --git a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue
index 2eea6a69bf..07747ed0cb 100644
--- a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue
+++ b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue
@@ -127,7 +127,7 @@ export default {
data () {
return {
loading: true,
- error: true,
+ error: false,
loadingData: false,
loadingCount: false,
loadingDelete: false,
diff --git a/dbrepo-ui/stores/cache.js b/dbrepo-ui/stores/cache.js
index 3574b24d7c..41059ba727 100644
--- a/dbrepo-ui/stores/cache.js
+++ b/dbrepo-ui/stores/cache.js
@@ -81,6 +81,14 @@ export const useCacheStore = defineStore('cache', {
console.error('Failed to reload view', error)
})
},
+ reloadSubset() {
+ const queryService = useQueryService()
+ queryService.findOne(this.subset.database_id, this.subset.id)
+ .then(subset => this.subset = subset)
+ .catch((error) => {
+ console.error('Failed to reload subset', error)
+ })
+ },
setRouteDatabase (databaseId) {
return new Promise((resolve, reject) => {
if (!databaseId) {
--
GitLab