diff --git a/dbrepo-data-service/Dockerfile b/dbrepo-data-service/Dockerfile index f4e2be5b967ca109e858d6f17099ec67d341ac12..4b45e9429050c08bcf4c3fd42b3d0ab259801657 100644 --- a/dbrepo-data-service/Dockerfile +++ b/dbrepo-data-service/Dockerfile @@ -8,7 +8,7 @@ LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at" COPY ./pom.xml ./ -RUN mvn -fn -B dependency:go-offline +RUN mvn -fn -B -q dependency:go-offline COPY --from=dependency /root/.m2/repository/at/tuwien /root/.m2/repository/at/tuwien @@ -18,7 +18,7 @@ COPY ./rest-service ./rest-service COPY ./services ./services # Make sure it compiles -RUN mvn clean package -DskipTests +RUN mvn -fn -B -q clean package -DskipTests ###### THIRD STAGE ###### FROM amazoncorretto:17-alpine3.19 AS runtime diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/RestEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/RestEndpoint.java index 333e0c8398e487d689de2669171d07a2f0c70c3c..45cea4371c74f61d3cca69d70b08e8341c5e78d3 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/RestEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/RestEndpoint.java @@ -45,6 +45,7 @@ public abstract class RestEndpoint { return UUID.fromString(user.getId()); } + /* FIXME: Heap may run OOM */ public List<Map<String, Object>> transform(Dataset<Row> dataset) { return dataset.collectAsList() .stream() diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java index 4dcfaf13a288c0d6c2415227552257396e450612..650652d62fe9a922c434276f89ec427d628236be 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java @@ -115,6 +115,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { database.setJdbcMethod(response.getHeaders().get("X-Type").get(0)); database.setUsername(response.getHeaders().get("X-Username").get(0)); database.setPassword(response.getHeaders().get("X-Password").get(0)); + database.setDatabase(database.getInternalName()); database.setHost(response.getHeaders().get("X-Host").get(0)); database.setPort(Integer.parseInt(response.getHeaders().get("X-Port").get(0))); database.setLastRetrieved(Instant.now()); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java index 7f5b63b21cc629a0087bf5737b66aa9c677aeaa5..d3af54816436699c413db11ff59cf213656b46a7 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java @@ -231,12 +231,6 @@ public interface MariaDbMapper { return statement.toString(); } - default String tableCreateDtoToSequenceName(at.tuwien.api.database.table.internal.TableCreateDto data) { - final String name = "seq_" + nameToInternalName(data.getName()) + "_id"; - log.trace("mapped table name {} to sequence name {}", data.getName(), name); - return name; - } - /** * Maps the desired data type to a MySQL string with the default MySQL 8 values for each * @@ -731,27 +725,30 @@ public interface MariaDbMapper { } } - default String selectRawSelectQuery(String query, Instant timestamp, Long page, Long size) { - query = query.toLowerCase(Locale.ROOT) - .trim(); - if (query.matches(";$")) { - /* remove last semicolon */ - query = query.substring(0, query.length() - 1); - } + default String defaultRawSelectQuery(String databaseName, String tableOrViewName, Instant timestamp, Long page, + Long size) { /* query check (this is enforced by the db also) */ - final StringBuilder statement = new StringBuilder("SELECT * FROM (") - .append(query) - .append(") FOR SYSTEM_TIME AS OF TIMESTAMP '") - .append(mariaDbFormatter.format(timestamp)) - .append("' as tbl"); + final StringBuilder statement = new StringBuilder("SELECT * FROM (SELECT * FROM `") + .append(databaseName) + .append("`.`") + .append(tableOrViewName) + .append("`"); + if (timestamp != null) { + statement.append(" FOR SYSTEM_TIME AS OF TIMESTAMP '") + .append(mariaDbFormatter.format(timestamp)) + .append("'"); + } + statement.append(" as tbl"); /* pagination */ - log.trace("pagination size/limit of {}", size); - statement.append(" LIMIT ") - .append(size); - log.trace("pagination page/offset of {}", page); - statement.append(" OFFSET ") - .append(page * size); - statement.append(";"); + if (size != null && page != null) { + log.trace("pagination size/limit of {}", size); + statement.append(" LIMIT ") + .append(size); + log.trace("pagination page/offset of {}", page); + statement.append(" OFFSET ") + .append(page * size); + } + statement.append(") as tbl2"); log.trace("mapped select query: {}", statement); return statement.toString(); } diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java index 1044869ad873b2eae3b1dd1ea8f92b19857dbd49..3a54b399d47e69263b355afb730e2800ca90a526 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DataConnector.java @@ -10,7 +10,6 @@ import org.springframework.stereotype.Service; public abstract class DataConnector<T extends CacheableDto> { public ComboPooledDataSource getDataSource(T entity) { - final long start = System.currentTimeMillis(); final ComboPooledDataSource dataSource = new ComboPooledDataSource(); dataSource.setJdbcUrl(getJdbcUrl(entity.getJdbcMethod(), entity.getHost(), entity.getPort(), entity.getDatabase())); @@ -25,7 +24,6 @@ public abstract class DataConnector<T extends CacheableDto> { } public ComboPooledDataSource getDataSource(T entity, String databaseName) { - final long start = System.currentTimeMillis(); final ComboPooledDataSource dataSource = new ComboPooledDataSource(); dataSource.setJdbcUrl(getJdbcUrl(entity.getJdbcMethod(), entity.getHost(), entity.getPort(), databaseName)); dataSource.setUser(entity.getUsername()); @@ -60,6 +58,7 @@ public abstract class DataConnector<T extends CacheableDto> { stringBuilder.append("/") .append(databaseName); } + log.trace("mapped jdbc url: {}", stringBuilder); return stringBuilder.toString(); } diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java index 68fb59cb5e02969e94553d4109b7ca141f07d67d..482f874624dc2694d305d7fcac7d2b32cc971b26 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java @@ -373,12 +373,16 @@ public class TableServiceMariaDbImpl extends DataConnector<TableDto> implements Long page, Long size, SortTypeDto sortDirection, String sortColumn) throws QueryMalformedException, TableNotFoundException { try { - final Properties properties = new Properties(); - properties.setProperty("user", database.getUsername()); - properties.setProperty("password", database.getPassword()); return sparkSession.read() - .jdbc(getSparkUrl(database.getJdbcMethod(), database.getHost(), database.getPort(), - database.getInternalName()), tableOrView, properties); + .format("jdbc") + .option("user", database.getUsername()) + .option("password", database.getPassword()) + .option("url", getSparkUrl(database.getJdbcMethod(), database.getHost(), database.getPort(), + database.getInternalName())) + .option("query", mariaDbMapper.defaultRawSelectQuery(database.getInternalName(), tableOrView, + timestamp, page, size)) + .load(); + } catch (Exception e) { if (e instanceof ExtendedAnalysisException exception) { if (exception.getSimpleMessage().contains("TABLE_OR_VIEW_NOT_FOUND")) { diff --git a/dbrepo-metadata-service/Dockerfile b/dbrepo-metadata-service/Dockerfile index 843c334a9acb3d24e061c42c72cb75c6068ef954..ddc20cb420764196c3a4b423294200e26eaa5bce 100644 --- a/dbrepo-metadata-service/Dockerfile +++ b/dbrepo-metadata-service/Dockerfile @@ -12,7 +12,7 @@ COPY ./rest-service/pom.xml ./rest-service/ COPY ./services/pom.xml ./services/ COPY ./test/pom.xml ./test/ -RUN mvn verify -B -fn +RUN mvn -fn -B dependency:go-offline COPY ./api ./api COPY ./entities ./entities @@ -24,7 +24,7 @@ COPY ./services ./services COPY ./test ./test # Make sure it compiles -RUN mvn clean install -DskipTests +RUN mvn -fn -B clean install -DskipTests ###### SECOND STAGE ###### FROM amazoncorretto:17-alpine3.19 AS runtime diff --git a/dbrepo-ui/components/subset/Results.vue b/dbrepo-ui/components/subset/Results.vue index e558186daf15c186dfdc085a844799815e3756a4..3948667518939fc6ae4d1a5d05502a81e3e14315 100644 --- a/dbrepo-ui/components/subset/Results.vue +++ b/dbrepo-ui/components/subset/Results.vue @@ -110,9 +110,13 @@ export default { this.id = id this.loadingExecute = false }) - .catch(({code}) => { + .catch(({code, message}) => { this.loadingExecute = false const toast = useToastInstance() + if (message) { + toast.error(message) + return + } if (typeof code !== 'string') { return } @@ -129,9 +133,13 @@ export default { this.id = id this.loadingExecute = false }) - .catch(({code}) => { + .catch(({code, message}) => { this.loadingExecute = false const toast = useToastInstance() + if (message) { + toast.error(message) + return + } if (typeof code !== 'string') { return } @@ -148,9 +156,13 @@ export default { this.id = id this.loadingExecute = false }) - .catch(({code}) => { + .catch(({code, message}) => { this.loadingExecute = false const toast = useToastInstance() + if (message) { + toast.error(message) + return + } if (typeof code !== 'string') { return } diff --git a/dbrepo-ui/components/subset/SubsetToolbar.vue b/dbrepo-ui/components/subset/SubsetToolbar.vue index 874e691c5f15766d7b4a0fb4809eb19058418c9c..d5f45e48e3a2d596977a37186a0076179bcd583f 100644 --- a/dbrepo-ui/components/subset/SubsetToolbar.vue +++ b/dbrepo-ui/components/subset/SubsetToolbar.vue @@ -35,7 +35,6 @@ variant="flat" class="mr-2" :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-content-save-outline' : null" - :disabled="!executionUTC" :to="`/database/${$route.params.database_id}/subset/${$route.params.subset_id}/persist`"> {{ ($vuetify.display.lgAndUp ? $t('toolbars.subset.pid.xl') + ' ' : '') + $t('toolbars.subset.pid.permanent') }} </v-btn> @@ -177,8 +176,9 @@ export default { this.loadingSave = true const queryService = useQueryService() queryService.update(this.$route.params.database_id, this.$route.params.subset_id, { persist: true }) - .then((subset) => { - this.subset = subset + .then(() => { + const cacheStore = useCacheStore() + cacheStore.reloadSubset() this.loadingSave = false }) .catch(() => { @@ -192,8 +192,10 @@ export default { this.loadingSave = true const queryService = useQueryService() queryService.update(this.$route.params.database_id, this.$route.params.subset_id, { persist: false }) - .then((subset) => { - this.subset = subset + .then(() => { + const cacheStore = useCacheStore() + cacheStore.reloadSubset() + this.loadingSave = false }) .catch(() => { this.loadingSave = false diff --git a/dbrepo-ui/composables/table-service.ts b/dbrepo-ui/composables/table-service.ts index ca757c7451d70c4bd1fffc36b72c6a23d7fdde58..45268d6295fc0ba55fdf268936a1614b350033e3 100644 --- a/dbrepo-ui/composables/table-service.ts +++ b/dbrepo-ui/composables/table-service.ts @@ -9,7 +9,7 @@ export const useTableService = (): any => { return new Promise<TableBriefDto>((resolve, reject) => { axios.get<TableBriefDto>(`/api/database/${databaseId}/table`) .then((response) => { - console.info('Found tables(s)') + console.info(`Found ${response.data.length} tables(s)`) resolve(response.data) }) .catch((error) => { @@ -25,7 +25,7 @@ export const useTableService = (): any => { return new Promise<TableDto>((resolve, reject) => { axios.get<TableDto>(`/api/database/${databaseId}/table/${tableId}`) .then((response) => { - console.info('Found table with id', tableId, 'in database with id', databaseId); + console.info('Found table'); resolve(response.data) }) .catch((error) => { @@ -41,7 +41,7 @@ export const useTableService = (): any => { return new Promise<ColumnDto>((resolve, reject) => { axios.put<ColumnDto>(`/api/database/${databaseId}/table/${tableId}/column/${columnId}`, data) .then((response) => { - console.info('Updated column with id', columnId, 'table with id', tableId, 'in database with id', databaseId); + console.info('Updated column'); resolve(response.data) }) .catch((error) => { @@ -57,7 +57,7 @@ export const useTableService = (): any => { return new Promise<TableDto>((resolve, reject) => { axios.put<TableDto>(`/api/database/${databaseId}/table/${tableId}`, data) .then((response) => { - console.info('Updated table with id', tableId, 'in database with id', databaseId); + console.info('Updated table'); resolve(response.data) }) .catch((error) => { @@ -73,7 +73,7 @@ export const useTableService = (): any => { return new Promise<ImportDto>((resolve, reject) => { axios.post<ImportDto>(`/api/database/${databaseId}/table/${tableId}/data/import`, data) .then((response) => { - console.info('Imported csv to table with id', tableId, 'in database with id', databaseId) + console.info('Imported csv to table') resolve(response.data) }) .catch((error) => { @@ -89,7 +89,7 @@ export const useTableService = (): any => { return new Promise<QueryResultDto>((resolve, reject) => { axios.get<QueryResultDto>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, page, size) }) .then((response) => { - console.info('Got data for table with id', tableId, 'in database with id', databaseId) + console.info('Got data for table') const result: QueryResultDto = { id: tableId, headers: response.headers['x-headers'] ? response.headers['x-headers'].split(',') : [], @@ -111,7 +111,7 @@ export const useTableService = (): any => { axios.head<void>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, null, null) }) .then((response: AxiosResponse<void>) => { const count: number = Number(response.headers['x-count']) - console.info('Found' + count + 'in table with id', tableId, 'in database with id', databaseId) + console.info(`Found ${count} tuple(s)`) resolve(count) }) .catch((error) => { @@ -134,7 +134,7 @@ export const useTableService = (): any => { return new Promise<QueryResultDto>((resolve, reject) => { axios.get<QueryResultDto>(`/api/database/${databaseId}/table/${tableId}/export`, config) .then((response) => { - console.info('Exported data for table with id', tableId, 'in database with id', databaseId) + console.info('Exported data for table') resolve(response.data) }) .catch((error) => { @@ -150,7 +150,7 @@ export const useTableService = (): any => { return new Promise<TableDto>((resolve, reject) => { axios.post<TableDto>(`/api/database/${databaseId}/table`, data) .then((response) => { - console.info('Created table in database with id', databaseId) + console.info('Created table') resolve(response.data) }) .catch((error: AxiosError) => { @@ -166,7 +166,7 @@ export const useTableService = (): any => { return new Promise<void>((resolve, reject) => { axios.delete<void>(`/api/database/${databaseId}/table/${tableId}`) .then((response) => { - console.info('Deleted table with id', tableId, 'in database with id', databaseId) + console.info('Deleted table') resolve(response.data) }) .catch((error) => { @@ -182,7 +182,7 @@ export const useTableService = (): any => { return new Promise<void>((resolve, reject) => { axios.delete<void>(`/api/database/${databaseId}/table/${tableId}`, {data}) .then((response) => { - console.info('Deleted tuple(s) in table with id', tableId, 'in database with id', databaseId) + console.info(`Deleted tuple(s)`) resolve(response.data) }) .catch((error) => { @@ -198,7 +198,7 @@ export const useTableService = (): any => { return new Promise<TableHistoryDto[]>((resolve, reject) => { axios.get<TableHistoryDto[]>(`/api/database/${databaseId}/table/${tableId}/history`) .then((response) => { - console.info('Loaded history of table with id', tableId, 'in database with id', databaseId) + console.info('Loaded history of table') resolve(response.data) }) .catch((error) => { @@ -214,7 +214,7 @@ export const useTableService = (): any => { return new Promise<TableColumnEntityDto[]>((resolve, reject) => { axios.get<TableColumnEntityDto[]>(`/api/database/${databaseId}/table/${tableId}/column/${columnId}/suggest`) .then((response) => { - console.info('Suggested semantic entities for table column with id', columnId, 'of table with id', tableId, 'of database with id', databaseId) + console.info('Suggested semantic entities') resolve(response.data) }) .catch((error) => { diff --git a/dbrepo-ui/nuxt.config.ts b/dbrepo-ui/nuxt.config.ts index 70084f3ccae68656ef23158b5a2ec29379d7e449..4b1833d8162e7e1494f4164fa90bb76e107f7635 100644 --- a/dbrepo-ui/nuxt.config.ts +++ b/dbrepo-ui/nuxt.config.ts @@ -75,8 +75,8 @@ export default defineNuxtConfig({ } }, api: { - client: 'https://dbrepo1.ec.tuwien.ac.at', - server: 'https://dbrepo1.ec.tuwien.ac.at', + client: 'http://localhost', + server: 'http://gateway-service', }, upload: { client: 'http://localhost/api/upload/files', diff --git a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue index f96b6e41758eb2a6e25d6ea3425dfb3c2ea043b3..e9719cd0efe49369d3735ab7f9d5a0e2acd22070 100644 --- a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue +++ b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue @@ -119,7 +119,7 @@ export default { }, }, mounted () { - this.loadResult() + this.loadSubset() }, methods: { loadSubset () { @@ -128,7 +128,9 @@ export default { queryService.findOne(this.$route.params.database_id, this.$route.params.subset_id) .then((subset) => { this.subset = subset - this.loadResult() + this.$refs.queryResults.reExecute(subset.id) + this.$refs.queryResults.reExecuteCount(subset.id) + this.loadingSubset = false }) .catch(() => { this.loadingSubset = false @@ -139,8 +141,6 @@ export default { }, loadResult () { if (this.subset) { - this.$refs.queryResults.reExecute(this.subset.id) - this.$refs.queryResults.reExecuteCount(this.subset.id) } }, download () { diff --git a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue index 2eea6a69bf58411451d4a33d06bb62282b8dd872..07747ed0cb50cdd950ee24dece9ff3c5faf6de06 100644 --- a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue +++ b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue @@ -127,7 +127,7 @@ export default { data () { return { loading: true, - error: true, + error: false, loadingData: false, loadingCount: false, loadingDelete: false, diff --git a/dbrepo-ui/stores/cache.js b/dbrepo-ui/stores/cache.js index 3574b24d7c6300b884f7874726254e3c805393b3..41059ba7270d93717998ac5b245e37d6ed239ee3 100644 --- a/dbrepo-ui/stores/cache.js +++ b/dbrepo-ui/stores/cache.js @@ -81,6 +81,14 @@ export const useCacheStore = defineStore('cache', { console.error('Failed to reload view', error) }) }, + reloadSubset() { + const queryService = useQueryService() + queryService.findOne(this.subset.database_id, this.subset.id) + .then(subset => this.subset = subset) + .catch((error) => { + console.error('Failed to reload subset', error) + }) + }, setRouteDatabase (databaseId) { return new Promise((resolve, reject) => { if (!databaseId) {