Merge branch 'dev' into 'master'

Dev

See merge request !319

.docker/.env

-# general
-BASE_URL=http://localhost
-ADMIN_EMAIL=support@localhost
-# password for the identity service admin user
-IDENTITY_SERVICE_ADMIN_PASSWORD=admin
-# password for the auth service admin user
-AUTH_SERVICE_ADMIN_PASSWORD=admin
-# passwords for root user of the databases
-METADATA_DB_PASSWORD=dbrepo
-DATA_DB_PASSWORD=dbrepo
-AUTH_DB_PASSWORD=dbrepo
-SEARCH_DB_PASSWORD=admin
-# storage service
-S3_SECRET_ACCESS_KEY=seaweedfsadmin
-# internal admin user, requires a change of the value of auth_ldap.dn_lookup_bind.password in dist/rabbitmq.conf
-SYSTEM_PASSWORD=admin

.gitignore

@@ -22,6 +22,7 @@ site/
 final/
 
 .$*
+dbrepo-somapper/
 
 # Notebooks
 .pytest_cache/

dbrepo-analyse-service/.gitignore

@@ -14,6 +14,9 @@ venv/
 .venv/
 env*
 
+# LLM
+*.bin
+
 # Test
 report.xml
 coverage.html

dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java

 package at.tuwien.endpoints;
 
 import at.tuwien.ExportResourceDto;
-import at.tuwien.api.database.DatabaseDto;
 import at.tuwien.api.database.internal.PrivilegedDatabaseDto;
 import at.tuwien.api.database.query.ExecuteStatementDto;
 import at.tuwien.api.database.query.QueryDto;

dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java

@@ -293,6 +293,9 @@ public interface MariaDbMapper {
                         .append(column.getInternalName())
                         .append("`) as std_dev FROM ")
                         .append(table));
+        if (statement.isEmpty()) {
+            return null;
+        }
         statement.append(";");
         log.trace("mapped select column statistic statement: {}", statement);
         return statement.toString();

dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java

@@ -30,6 +30,8 @@ import java.sql.*;
 import java.time.Instant;
 import java.util.*;
 
+import static org.apache.jena.sparql.util.QueryExecUtils.executeQuery;
+
 @Log4j2
 @Service
 public class TableServiceMariaDbImpl extends HibernateConnector implements TableService {
@@ -96,7 +98,12 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
         try {
             /* obtain statistic */
             final long start = System.currentTimeMillis();
-            final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.tableColumnStatisticsSelectRawQuery(table.getColumns(), table.getInternalName()))
+            final String query = mariaDbMapper.tableColumnStatisticsSelectRawQuery(table.getColumns(), table.getInternalName());
+            if (query == null) {
+                log.debug("table {}.{} does not have columns that can be analysed for statistical properties (i.e. no numeric columns)", table.getDatabase().getInternalName(), table.getInternalName());
+                statistic = null;
+            } else {
+                final ResultSet resultSet = connection.prepareStatement(query)
                         .executeQuery();
                 log.debug("executed statement in {} ms", System.currentTimeMillis() - start);
                 statistic = dataMapper.resultSetToTableStatistic(resultSet);
@@ -105,6 +112,14 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
                 statistic.setDataLength(tmpTable.getDataLength());
                 statistic.setMaxDataLength(tmpTable.getMaxDataLength());
                 statistic.setRows(tmpTable.getNumRows());
+                /* add to statistic dto */
+                table.getColumns()
+                        .stream()
+                        .filter(column -> !MariaDbUtil.numericDataTypes.contains(column.getColumnType()))
+                        .forEach(column -> statistic.getColumns().put(column.getInternalName(), new ColumnStatisticDto()));
+                log.info("Obtained statistics for the table and {} column(s)", statistic.getColumns().size());
+                log.trace("obtained statistics: {}", statistic);
+            }
         } catch (SQLException e) {
             connection.rollback();
             log.error("Failed to obtain column statistics: {}", e.getMessage());
@@ -112,12 +127,6 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table
         } finally {
             dataSource.close();
         }
-        table.getColumns()
-                .stream()
-                .filter(column -> !MariaDbUtil.numericDataTypes.contains(column.getColumnType()))
-                .forEach(column -> statistic.getColumns().put(column.getInternalName(), new ColumnStatisticDto()));
-        log.info("Obtained statistics for the table and {} column(s)", statistic.getColumns().size());
-        log.trace("obtained statistics: {}", statistic);
         return statistic;
     }
 

dbrepo-gateway-service/dbrepo.conf

-client_max_body_size 2G;
+client_max_body_size 20G;
 
 resolver 127.0.0.11 valid=30s; # docker dns
 

docker-compose.yml

@@ -312,6 +312,7 @@ services:
       network: host
     environment:
       NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}"
+      NUXT_PUBLIC_API_SERVER: "${BASE_URL:-http://localhost}"
       NUXT_PUBLIC_UPLOAD_CLIENT: "${BASE_URL:-http://localhost}/api/upload/files"
     depends_on:
       dbrepo-search-service:
@@ -460,8 +461,8 @@ services:
       - "-s3-endpoint=${STORAGE_ENDPOINT:-http://storage-service:9000}"
       - "-s3-bucket=dbrepo"
     environment:
-      AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-seaweedfsadmin}"
-      AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-seaweedfsadmin}"
+      AWS_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
+      AWS_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
       AWS_REGION: "${STORAGE_REGION_NAME:-default}"
     depends_on:
       dbrepo-storage-service:
@@ -487,22 +488,22 @@ services:
     volumes:
       - "${SHARED_VOLUME:-/tmp}:/tmp"
     environment:
-      AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-admin}
-      AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-admin}
-      AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
-      AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
-      AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
-      BROKER_EXCHANGE_NAME: ${BROKER_EXCHANGE_NAME:-dbrepo}
-      BROKER_QUEUE_NAME: ${BROKER_QUEUE_NAME:-dbrepo}
+      AUTH_SERVICE_ADMIN: "${AUTH_SERVICE_ADMIN:-admin}"
+      AUTH_SERVICE_ADMIN_PASSWORD: "${AUTH_SERVICE_ADMIN_PASSWORD:-admin}"
+      AUTH_SERVICE_CLIENT: "${AUTH_SERVICE_CLIENT:-dbrepo-client}"
+      AUTH_SERVICE_CLIENT_SECRET: "${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}"
+      AUTH_SERVICE_ENDPOINT: "${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}"
+      BROKER_EXCHANGE_NAME: "${BROKER_EXCHANGE_NAME:-dbrepo}"
+      BROKER_QUEUE_NAME: "${BROKER_QUEUE_NAME:-dbrepo}"
       BROKER_HOST: "${BROKER_ENDPOINT:-broker-service}"
       BROKER_PASSWORD: "${SYSTEM_PASSWORD:-admin}"
       BROKER_PORT: ${BROKER_PORT:-5672}
-      BROKER_SERVICE_ENDPOINT: ${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}
+      BROKER_SERVICE_ENDPOINT: "${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}"
       BROKER_USERNAME: "${SYSTEM_USERNAME:-admin}"
       BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}"
       CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000}
-      EXCHANGE_NAME: ${EXCHANGE_NAME:-dbrepo}
-      METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
+      EXCHANGE_NAME: "${EXCHANGE_NAME:-dbrepo}"
+      METADATA_SERVICE_ENDPOINT: "${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}"
       GRANT_DEFAULT_READ: "${GRANT_DEFAULT_READ:-SELECT}"
       GRANT_DEFAULT_WRITE: "${GRANT_DEFAULT_WRITE:-SELECT, CREATE, CREATE VIEW, CREATE ROUTINE, CREATE TEMPORARY TABLES, LOCK TABLES, INDEX, TRIGGER, INSERT, UPDATE, DELETE}"
       JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"

lib/python/dbrepo/RestClient.py

@@ -2,6 +2,7 @@ import os
 import sys
 import logging
 import time
+import datetime
 
 import requests
 from pydantic import TypeAdapter
@@ -1556,7 +1557,7 @@ class RestClient:
                                 f'201 (CREATED): {response.text}')
 
     def create_subset(self, database_id: int, query: str, page: int = 0, size: int = 10,
-                      df: bool = False) -> Result | DataFrame:
+                      timestamp: datetime.datetime = None, df: bool = False) -> Result | DataFrame:
         """
         Executes a SQL query in a database where the current user has at least read access with given database id. The
         result set can be paginated with setting page and size (both). Historic data can be queried by setting
@@ -1566,6 +1567,7 @@ class RestClient:
         :param query: The query statement.
         :param page: The result pagination number. Optional. Default: 0.
         :param size: The result pagination size. Optional. Default: 10.
+        :param timestamp: The timestamp at which the data validity is set. Optional. Default: <current timestamp>.
         :param df: If true, the result is returned as Pandas DataFrame. Optional. Default: False.
 
         :returns: The result set, if successful.
@@ -1581,6 +1583,11 @@ class RestClient:
         url = f'/api/database/{database_id}/subset'
         if page is not None and size is not None:
             url += f'?page={page}&size={size}'
+            if timestamp is not None:
+                url += f'&timestamp={timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")}'
+        else:
+            if timestamp is not None:
+                url += f'?timestamp={timestamp.strftime("%Y-%m-%dT%H:%M:%SZ")}'
         response = self._wrapper(method="post", url=url, force_auth=True, headers={"Accept": "application/json"},
                                  payload=ExecuteQuery(statement=query))
         if response.status_code == 201: