Fixed the connection pool for database service

.jupyter/deposit.ipynb

@@ -324,7 +324,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": []
   }

.jupyter/feature_extract.ipynb

@@ -893,7 +893,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [],
    "source": []
   }

fda-database-service/rest-service/src/main/java/at/tuwien/endpoints/ContainerDatabaseEndpoint.java

@@ -8,7 +8,6 @@ import at.tuwien.entities.database.Database;
 import at.tuwien.exception.*;
 import at.tuwien.mapper.DatabaseMapper;
 import at.tuwien.service.MessageQueueService;
-import at.tuwien.service.QueryStoreService;
 import at.tuwien.service.impl.MariaDbServiceImpl;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.security.SecurityRequirement;
@@ -34,15 +33,13 @@ public class ContainerDatabaseEndpoint {
 
     private final DatabaseMapper databaseMapper;
     private final MariaDbServiceImpl databaseService;
-    private final QueryStoreService queryStoreService;
     private final MessageQueueService messageQueueService;
 
     @Autowired
     public ContainerDatabaseEndpoint(DatabaseMapper databaseMapper, MariaDbServiceImpl databaseService,
-                                     QueryStoreService queryStoreService, MessageQueueService messageQueueService) {
+                                     MessageQueueService messageQueueService) {
         this.databaseMapper = databaseMapper;
         this.databaseService = databaseService;
-        this.queryStoreService = queryStoreService;
         this.messageQueueService = messageQueueService;
     }
 
@@ -78,11 +75,10 @@ public class ContainerDatabaseEndpoint {
                                                    @Valid @RequestBody DatabaseCreateDto createDto,
                                                    Principal principal)
             throws ImageNotSupportedException, ContainerNotFoundException, DatabaseMalformedException,
-            AmqpException, ContainerConnectionException, UserNotFoundException, QueryStoreException,
-            DatabaseNotFoundException, DatabaseNameExistsException {
+            AmqpException, ContainerConnectionException, UserNotFoundException,
+            DatabaseNotFoundException, DatabaseNameExistsException, DatabaseConnectionException {
         final Database database = databaseService.create(containerId, createDto, principal);
         messageQueueService.createExchange(database, principal);
-        queryStoreService.create(containerId, database.getId());
         return ResponseEntity.status(HttpStatus.CREATED)
                 .body(databaseMapper.databaseToDatabaseBriefDto(database));
     }
@@ -128,7 +124,7 @@ public class ContainerDatabaseEndpoint {
                                     @NotBlank @PathVariable Long databaseId,
                                     Principal principal) throws DatabaseNotFoundException,
             ImageNotSupportedException, DatabaseMalformedException, AmqpException, ContainerConnectionException,
-            ContainerNotFoundException {
+            ContainerNotFoundException, DatabaseConnectionException {
         final Database database = databaseService.findById(containerId, databaseId);
         messageQueueService.deleteExchange(database);
         databaseService.delete(containerId, databaseId, principal);

fda-database-service/rest-service/src/main/resources/querystore.sql

+-- SEQUENCES
+CREATE SEQUENCE `qs_tables_seq`;
+CREATE SEQUENCE `qs_columns_seq`;
+CREATE SEQUENCE `qs_columns_seq`;
+
+-- TABLES
+CREATE TABLE `qs_queries`
+(
+    `id`            bigint       not null primary key default nextval(`qs_queries_seq`),
+    `cid`           bigint       not null,
+    `created`       datetime     not null,
+    `created_by`    bigint       not null,
+    `dbid`          bigint       not null,
+    `execution`     datetime     not null,
+    `last_modified` datetime     not null,
+    `query`         text         not null,
+    `query_hash`    varchar(255) not null,
+    `result_hash`   varchar(255),
+    `result_number` bigint
+);
+CREATE TABLE `qs_tables`
+(
+    `id`            bigint   not null primary key default nextval(`qs_tables_seq`),
+    `created`       datetime not null,
+    `dbid`          bigint   not null,
+    `last_modified` datetime
+);
+CREATE TABLE `qs_columns`
+(
+    `id`            bigint   not null primary key default nextval(`qs_columns_seq`),
+    `created`       datetime not null,
+    `dbid`          bigint   not null,
+    `tid`           bigint   not null,
+    `last_modified` datetime
+);
\ No newline at end of file

fda-database-service/services/src/main/java/at/tuwien/mapper/DatabaseMapper.java

@@ -7,6 +7,7 @@ import at.tuwien.api.database.LanguageTypeDto;
 import at.tuwien.api.user.UserDetailsDto;
 import at.tuwien.entities.database.Database;
 import at.tuwien.entities.database.LanguageType;
+import at.tuwien.querystore.Query;
 import org.apache.http.auth.BasicUserPrincipal;
 import org.mapstruct.Mapper;
 import org.mapstruct.Mapping;

fda-database-service/services/src/main/java/at/tuwien/service/DatabaseService.java

@@ -63,7 +63,7 @@ public interface DatabaseService {
      */
     void delete(Long id, Long databaseId, Principal principal)
             throws DatabaseNotFoundException, ImageNotSupportedException,
-            DatabaseMalformedException, AmqpException, ContainerConnectionException, ContainerNotFoundException;
+            DatabaseMalformedException, AmqpException, ContainerConnectionException, ContainerNotFoundException, DatabaseConnectionException;
 
     /**
      * Creates a new database with minimal metadata in the metadata database and creates a new database on the container.
@@ -80,7 +80,7 @@ public interface DatabaseService {
      */
     Database create(Long id, DatabaseCreateDto createDto, Principal principal)
             throws ImageNotSupportedException, ContainerNotFoundException,
-            DatabaseMalformedException, AmqpException, ContainerConnectionException, UserNotFoundException, DatabaseNameExistsException;
+            DatabaseMalformedException, AmqpException, ContainerConnectionException, UserNotFoundException, DatabaseNameExistsException, DatabaseConnectionException;
 
     /**
      * Updates the database metadata.

fda-database-service/services/src/main/java/at/tuwien/service/QueryStoreService.java

-package at.tuwien.service;
-
-import at.tuwien.exception.DatabaseNotFoundException;
-import at.tuwien.exception.ImageNotSupportedException;
-import at.tuwien.exception.QueryStoreException;
-
-public interface QueryStoreService {
-
-    /**
-     * Creates the query store by executing a query, the Hibernate session is configured to automatically create the necessary table.
-     *
-     * @param containerId The container id.
-     * @param databaseId  The database id.
-     * @throws DatabaseNotFoundException  The database was not found in the metadata database.
-     * @throws ImageNotSupportedException The image is not supported, currently we only support MariaDB.
-     * @throws QueryStoreException        The query store failed to create.
-     */
-    void create(Long containerId, Long databaseId) throws DatabaseNotFoundException,
-            ImageNotSupportedException, QueryStoreException;
-}

fda-database-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java

@@ -5,18 +5,18 @@ import at.tuwien.entities.container.image.ContainerImage;
 import at.tuwien.entities.container.image.ContainerImageEnvironmentItem;
 import at.tuwien.entities.container.image.ContainerImageEnvironmentItemType;
 import at.tuwien.entities.database.Database;
-import at.tuwien.querystore.Column;
-import at.tuwien.querystore.Query;
-import at.tuwien.querystore.Table;
+import at.tuwien.exception.DatabaseConnectionException;
+import com.mchange.v2.c3p0.ComboPooledDataSource;
 import lombok.extern.log4j.Log4j2;
 import org.hibernate.Session;
-import org.hibernate.SessionFactory;
-import org.hibernate.cfg.Configuration;
 import org.hibernate.query.NativeQuery;
 import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
 
 import javax.persistence.PersistenceException;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -24,54 +24,68 @@ import java.util.stream.Collectors;
 @Service
 public abstract class HibernateConnector {
 
-    protected static Session getCurrentSession(ContainerImage image, Container container) {
-        return getCurrentSession(image, container, null);
+    protected static Connection getConnection(ContainerImage image, Container container) throws DatabaseConnectionException {
+        return getConnection(image, container, null);
     }
 
-    protected static Session getCurrentSession(ContainerImage image, Container container, Database database) {
+    protected static Connection getConnection(ContainerImage image, Container container, Database database) throws DatabaseConnectionException {
+        final ComboPooledDataSource dataSource = new ComboPooledDataSource();
         final String url = "jdbc:" + image.getJdbcMethod() + "://" + container.getInternalName() + "/" + (database != null ? database.getInternalName() : "");
+        dataSource.setJdbcUrl(url);
         final String username = image.getEnvironment()
                 .stream()
                 .filter(e -> e.getType().equals(ContainerImageEnvironmentItemType.PRIVILEGED_USERNAME))
                 .map(ContainerImageEnvironmentItem::getValue)
                 .collect(Collectors.toList())
                 .get(0);
+        dataSource.setUser(username);
         final String password = image.getEnvironment()
                 .stream()
                 .filter(e -> e.getType().equals(ContainerImageEnvironmentItemType.PRIVILEGED_PASSWORD))
                 .map(ContainerImageEnvironmentItem::getValue)
                 .collect(Collectors.toList())
                 .get(0);
-        final Configuration config = new Configuration();
-        config.configure("mariadb_hibernate.cfg.xml");
-        config.setProperty("hibernate.connection.url", url);
-        config.setProperty("hibernate.connection.username", username);
-        config.setProperty("hibernate.connection.password", password);
-        config.setProperty("hibernate.connection.driver_class", image.getDriverClass());
-        config.setProperty("hibernate.dialect", image.getDialect());
-        final SessionFactory sessionFactory = config.buildSessionFactory();
-        Session session = sessionFactory.getCurrentSession();
-        if (!session.isOpen()) {
-            log.warn("Session is closed, opening...");
-            session = sessionFactory.openSession();
+        dataSource.setPassword(password);
+        dataSource.setInitialPoolSize(5);
+        dataSource.setMinPoolSize(5);
+        dataSource.setAcquireIncrement(5);
+        dataSource.setMaxPoolSize(20);
+        dataSource.setMaxStatements(100);
+        final Connection connection;
+        try {
+            connection = dataSource.getConnection();
+        } catch (SQLException e) {
+            log.error("Failed to connect to the database");
+            log.debug("failed to connect to the database {}", database);
+            throw new DatabaseConnectionException("Failed to connect to the database");
         }
-        return session;
+        return connection;
     }
 
-    protected static Long activeConnection(Session session) {
-        final NativeQuery<?> nativeQuery = session.createSQLQuery("SHOW STATUS LIKE 'threads_connected'");
-        final List<?> result;
+    protected static Long activeConnection(Connection connection) throws DatabaseConnectionException {
+        final ResultSet resultSet = execute(connection, "SHOW STATUS LIKE 'threads_connected'");
         try {
-            result = nativeQuery.getResultList();
-        } catch (PersistenceException e) {
-            log.error("Failed to collect number of used connections");
-            /* ignore */
-            return null;
+            if (resultSet.next()) {
+                return resultSet.getLong(2);
+            }
+        } catch (SQLException e) {
+            log.error("Failed to determine active connections");
+            throw new DatabaseConnectionException("Failed to determine active connections", e);
         }
-        final Object[] row = (Object[]) result.get(0);
-        log.debug("current number of connections: {}", Long.parseLong(String.valueOf(row[1])));
-        return Long.parseLong(String.valueOf(row[1]));
+        log.error("Failed to determine active connections");
+        throw new DatabaseConnectionException("Failed to determine active connections");
     }
 
+    protected static ResultSet execute(Connection connection, String statement) throws DatabaseConnectionException {
+        final PreparedStatement preparedStatement;
+        try {
+            preparedStatement = connection.prepareStatement(statement);
+            return preparedStatement.executeQuery();
+        } catch (SQLException e) {
+            log.error("Failed to execute statement");
+            log.debug("failed to execute statement {}", statement);
+            throw new DatabaseConnectionException("Failed to execute statement", e);
+        }
+    }
 
 }

fda-database-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java

@@ -27,6 +27,9 @@ import org.springframework.transaction.annotation.Transactional;
 
 import javax.persistence.PersistenceException;
 import java.security.Principal;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.time.Instant;
 import java.util.List;
 import java.util.Optional;
@@ -101,29 +104,16 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
     @Transactional
     public void delete(Long containerId, Long databaseId, Principal principal) throws DatabaseNotFoundException,
             ImageNotSupportedException, DatabaseMalformedException, ContainerConnectionException, AmqpException,
-            ContainerNotFoundException {
+            ContainerNotFoundException, DatabaseConnectionException {
         final Container container = containerService.find(containerId);
         final Database database = findPublicOrMineById(containerId, databaseId, principal);
         if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
             throw new ImageNotSupportedException("Currently only MariaDB is supported");
         }
         /* run query */
-        final Session session = getCurrentSession(container.getImage(), container, database);
-        final Transaction transaction = session.beginTransaction();
-        final NativeQuery<?> query = session.createSQLQuery(databaseMapper.databaseToRawDeleteDatabaseQuery(database));
-        try {
-            log.debug("query affected {} rows", query.executeUpdate());
-            activeConnection(session);
-            transaction.commit();
-        } catch (ServiceException e) {
-            log.error("Failed to delete database.");
-            session.close();
-            throw new DatabaseMalformedException("Failed to delete database", e);
-        } finally {
-            if (session.isOpen()) {
-                session.close();
-            }
-        }
+        final Connection connection = getConnection(container.getImage(), container, database);
+        execute(connection, databaseMapper.databaseToRawDeleteDatabaseQuery(database));
+        activeConnection(connection);
         database.setDeleted(Instant.now()) /* method has void, only for debug logs */;
         /* save in metadata database */
         databaseRepository.deleteById(databaseId);
@@ -135,7 +125,8 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
     @Transactional
     public Database create(Long containerId, DatabaseCreateDto createDto, Principal principal)
             throws ImageNotSupportedException, ContainerNotFoundException,
-            DatabaseMalformedException, AmqpException, ContainerConnectionException, UserNotFoundException, DatabaseNameExistsException {
+            DatabaseMalformedException, AmqpException, ContainerConnectionException, UserNotFoundException,
+            DatabaseNameExistsException, DatabaseConnectionException {
         final Container container = containerService.find(containerId);
         if (container.getDatabases().size() != 0) {
             log.error("Currently we only support one database per container.");
@@ -146,31 +137,13 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
         database.setName(createDto.getName());
         database.setInternalName(databaseMapper.nameToInternalName(database.getName()));
         database.setContainer(container);
-        /* run query */
-        final Session session = getCurrentSession(container.getImage(), container);
-        final Transaction transaction = session.beginTransaction();
-        final NativeQuery<?> query = session.createSQLQuery(databaseMapper.databaseToRawCreateDatabaseQuery(database));
-        try {
-            log.debug("query affected {} rows", query.executeUpdate());
-        } catch (PersistenceException e) {
-            log.error("Failed to create database");
-            log.debug("failed to create database: {}", e.getMessage());
-            session.close();
-            throw new DatabaseNameExistsException("Failed to create database", e);
-        }
-        final NativeQuery<?> grant = session.createSQLQuery(databaseMapper.imageToRawGrantReadonlyAccessQuery());
-        try {
-            log.debug("grant affected {} rows", grant.executeUpdate());
-            activeConnection(session);
-            transaction.commit();
-        } catch (PersistenceException e) {
-            log.error("Failed to grant privileges.");
-            throw new DatabaseMalformedException("Failed to grant privileges", e);
-        } finally {
-            if (session.isOpen()) {
-                session.close();
-            }
-        }
+        /* create database */
+        final Connection connection = getConnection(container.getImage(), container);
+        execute(connection, databaseMapper.databaseToRawCreateDatabaseQuery(database));
+        log.debug("active database connections {}", activeConnection(connection));
+        /* grant read-only access */
+        execute(connection, databaseMapper.imageToRawGrantReadonlyAccessQuery());
+        log.debug("active database connections {}", activeConnection(connection));
         /* save in metadata database */
         database.setExchange(amqpMapper.exchangeName(database));
         database.setDescription(createDto.getDescription());

fda-database-service/services/src/main/java/at/tuwien/service/impl/QueryStoreServiceImpl.java

-package at.tuwien.service.impl;
-
-import at.tuwien.entities.database.Database;
-import at.tuwien.exception.DatabaseNotFoundException;
-import at.tuwien.exception.ImageNotSupportedException;
-import at.tuwien.exception.QueryStoreException;
-import at.tuwien.service.DatabaseService;
-import at.tuwien.service.QueryStoreService;
-import lombok.extern.log4j.Log4j2;
-import org.hibernate.Session;
-import org.hibernate.Transaction;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import javax.persistence.PersistenceException;
-
-@Log4j2
-@Service
-public class QueryStoreServiceImpl extends HibernateConnector implements QueryStoreService {
-
-    private final DatabaseService databaseService;
-
-    @Autowired
-    public QueryStoreServiceImpl(DatabaseService databaseService) {
-        this.databaseService = databaseService;
-    }
-
-    @Override
-    public void create(Long containerId, Long databaseId) throws DatabaseNotFoundException,
-            ImageNotSupportedException, QueryStoreException {
-        /* find */
-        final Database database = databaseService.findById(containerId, databaseId);
-        if (!database.getContainer().getImage().getRepository().equals("mariadb")) {
-            throw new ImageNotSupportedException("Currently only MariaDB is supported");
-        }
-        /* run query */
-        final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
-        final Transaction transaction = session.beginTransaction();
-        /* use jpq to select all */
-        final org.hibernate.query.Query<at.tuwien.querystore.Query> queries = session.createQuery("select q from Query q",
-                at.tuwien.querystore.Query.class);
-        try {
-            queries.getResultList();
-            activeConnection(session);
-            transaction.commit();
-        } catch (PersistenceException e) {
-            log.error("Failed to find all queries");
-            session.close();
-            throw new QueryStoreException("Failed to find all queries");
-        } finally {
-            if (session.isOpen()) {
-                session.close();
-            }
-        }
-    }
-}