Skip to content
Snippets Groups Projects
Commit 10a2c221 authored by Martin Weise's avatar Martin Weise
Browse files

Updated the connection pool for the table service

parent 310c1091
Branches
Tags
2 merge requests!81New stable release,!80Multiple features connected with user management and ownership of databases
This commit is part of merge request !81. Comments created here will be created in the context of that merge request.
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 124 additions and 2081 deletions
.jupyter/test.ipynb
+ 54
− 54
Edit
View file @ 10a2c221
%% Cell type:markdown id: tags:
# Test Jupyter Notebook
%% Cell type:code id: tags:
``` python
import time
import os
import shutil
import uuid
from api_authentication.api.authentication_endpoint_api import AuthenticationEndpointApi
from api_authentication.api.user_endpoint_api import UserEndpointApi
from api_container.api.container_endpoint_api import ContainerEndpointApi
from api_database.api.container_database_endpoint_api import ContainerDatabaseEndpointApi
from api_table.api.table_endpoint_api import TableEndpointApi
from api_query.api.table_data_endpoint_api import TableDataEndpointApi
from api_query.api.query_endpoint_api import QueryEndpointApi
from api_identifier.api.identifier_endpoint_api import IdentifierEndpointApi
from api_identifier.api.persistence_endpoint_api import PersistenceEndpointApi
authentication = AuthenticationEndpointApi()
user = UserEndpointApi()
container = ContainerEndpointApi()
database = ContainerDatabaseEndpointApi()
table = TableEndpointApi()
query = QueryEndpointApi()
data = TableDataEndpointApi()
identifier = IdentifierEndpointApi()
persistence = PersistenceEndpointApi()
username = "user"
password = "user"
email = "someone@example.com"
```
%% Cell type:markdown id: tags:
Create user
%% Cell type:code id: tags:
``` python
response = user.register({
"username": username,
"password": password,
"email": email
})
print(response)
```
%% Output
{'affiliation': None,
'authorities': [{'authority': 'ROLE_RESEARCHER'}],
'containers': None,
'databases': None,
'email': 'someone@example.com',
'email_verified': False,
'firstname': None,
'id': 2,
'identifiers': None,
'lastname': None,
'orcid': None,
'titles_after': None,
'titles_before': None,
'username': 'user'}
%% Cell type:markdown id: tags:
Create token
%% Cell type:code id: tags:
``` python
response = authentication.authenticate_user1({
"username": username,
"password": password
})
user_id = response.id
token = response.token
container.api_client.default_headers = {"Authorization": "Bearer " + token}
database.api_client.default_headers = {"Authorization": "Bearer " + token}
table.api_client.default_headers = {"Authorization": "Bearer " + token}
data.api_client.default_headers = {"Authorization": "Bearer " + token}
query.api_client.default_headers = {"Authorization": "Bearer " + token}
identifier.api_client.default_headers = {"Authorization": "Bearer " + token}
user.api_client.default_headers = {"Authorization": "Bearer " + token}
persistence.api_client.default_headers = {"Authorization": "Bearer " + token}
```
%% Cell type:markdown id: tags:
Create container
%% Cell type:code id: tags:
``` python
response = container.create1({
"name": "Airquality " + str(uuid.uuid1()),
"repository": "mariadb",
"tag": "10.5"
})
container_id = response.id
print(response)
```
%% Output
{'created': datetime.datetime(2022, 7, 23, 11, 31, 39, 906000, tzinfo=tzutc()),
{'created': datetime.datetime(2022, 7, 23, 12, 10, 34, 731000, tzinfo=tzutc()),
'creator': {'affiliation': None,
'authorities': None,
'email': 'someone@example.com',
'email_verified': False,
'firstname': None,
'id': 2,
'lastname': None,
'orcid': None,
'titles_after': None,
'titles_before': None,
'username': 'user'},
'hash': 'afa5b9a1ef1d07f09a90f6c76c696974a194376b5d3bae48e3a442056dc30ce2',
'hash': 'a7ac6834e9c29d93a0c7049707bbb1350cbd87d9d2836fc4b19dfadc8bbe9724',
'id': 1,
'internal_name': 'fda-userdb-airquality-0468fb50-0a7b-11ed-95b6-4f6e5b6c5022',
'internal_name': 'fda-userdb-airquality-741b9886-0a80-11ed-95b6-4f6e5b6c5022',
'is_public': None,
'name': 'Airquality 0468fb50-0a7b-11ed-95b6-4f6e5b6c5022'}
'name': 'Airquality 741b9886-0a80-11ed-95b6-4f6e5b6c5022'}
%% Cell type:markdown id: tags:
Start container
%% Cell type:code id: tags:
``` python
response = container.modify({
"action": "start"
}, container_id)
time.sleep(5)
print(response)
```
%% Output
{'created': datetime.datetime(2022, 7, 23, 11, 31, 39, 906000, tzinfo=tzutc()),
{'created': datetime.datetime(2022, 7, 23, 12, 10, 34, 731000, tzinfo=tzutc()),
'creator': {'affiliation': None,
'authorities': None,
'email': 'someone@example.com',
'email_verified': False,
'firstname': None,
'id': 2,
'lastname': None,
'orcid': None,
'titles_after': None,
'titles_before': None,
'username': 'user'},
'hash': 'afa5b9a1ef1d07f09a90f6c76c696974a194376b5d3bae48e3a442056dc30ce2',
'hash': 'a7ac6834e9c29d93a0c7049707bbb1350cbd87d9d2836fc4b19dfadc8bbe9724',
'id': 1,
'internal_name': 'fda-userdb-airquality-0468fb50-0a7b-11ed-95b6-4f6e5b6c5022',
'internal_name': 'fda-userdb-airquality-741b9886-0a80-11ed-95b6-4f6e5b6c5022',
'is_public': None,
'name': 'Airquality 0468fb50-0a7b-11ed-95b6-4f6e5b6c5022'}
'name': 'Airquality 741b9886-0a80-11ed-95b6-4f6e5b6c5022'}
%% Cell type:markdown id: tags:
Create database
%% Cell type:code id: tags:
``` python
response = database.create({
"name": "Airquality " + str(uuid.uuid1()),
"description": "Hourly measurements in Zürich, Switzerland",
"is_public": True
}, container_id)
database_id = response.id
print(response)
```
%% Output
{'container': {'created': datetime.datetime(2022, 7, 23, 11, 31, 39, 906000, tzinfo=tzutc()),
{'container': {'created': datetime.datetime(2022, 7, 23, 12, 10, 34, 731000, tzinfo=tzutc()),
'creator': {'affiliation': None,
'authorities': None,
'email': 'someone@example.com',
'email_verified': False,
'firstname': None,
'id': 2,
'lastname': None,
'orcid': None,
'titles_after': None,
'titles_before': None,
'username': 'user'},
'hash': 'afa5b9a1ef1d07f09a90f6c76c696974a194376b5d3bae48e3a442056dc30ce2',
'hash': 'a7ac6834e9c29d93a0c7049707bbb1350cbd87d9d2836fc4b19dfadc8bbe9724',
'id': 1,
'internal_name': 'fda-userdb-airquality-0468fb50-0a7b-11ed-95b6-4f6e5b6c5022',
'internal_name': 'fda-userdb-airquality-741b9886-0a80-11ed-95b6-4f6e5b6c5022',
'is_public': None,
'name': 'Airquality 0468fb50-0a7b-11ed-95b6-4f6e5b6c5022'},
'created': datetime.datetime(2022, 7, 23, 11, 31, 57, 23000, tzinfo=tzutc()),
'name': 'Airquality 741b9886-0a80-11ed-95b6-4f6e5b6c5022'},
'created': datetime.datetime(2022, 7, 23, 12, 10, 48, 550000, tzinfo=tzutc()),
'creator': {'affiliation': None,
'authorities': None,
'email': 'someone@example.com',
'email_verified': False,
'firstname': None,
'id': 2,
'lastname': None,
'orcid': None,
'titles_after': None,
'titles_before': None,
'username': 'user'},
'description': 'Hourly measurements in Zürich, Switzerland',
'engine': 'mariadb:10.5',
'id': 1,
'is_public': True,
'name': 'Airquality 0e9a48ae-0a7b-11ed-95b6-4f6e5b6c5022'}
'name': 'Airquality 7c5e7aea-0a80-11ed-95b6-4f6e5b6c5022'}
%% Cell type:markdown id: tags:
Inspect database
%% Cell type:code id: tags:
``` python
response = database.find_by_id(container_id, database_id)
database_exchange = response.exchange
print(response)
```
%% Output
{'contact': None,
'container': {'created': datetime.datetime(2022, 7, 23, 11, 31, 39, 906000, tzinfo=tzutc()),
'container': {'created': datetime.datetime(2022, 7, 23, 12, 10, 34, 731000, tzinfo=tzutc()),
'databases': None,
'hash': 'afa5b9a1ef1d07f09a90f6c76c696974a194376b5d3bae48e3a442056dc30ce2',
'hash': 'a7ac6834e9c29d93a0c7049707bbb1350cbd87d9d2836fc4b19dfadc8bbe9724',
'id': 1,
'image': {'id': 1, 'repository': 'mariadb', 'tag': '10.5'},
'internal_name': 'fda-userdb-airquality-0468fb50-0a7b-11ed-95b6-4f6e5b6c5022',
'internal_name': 'fda-userdb-airquality-741b9886-0a80-11ed-95b6-4f6e5b6c5022',
'ip_address': None,
'is_public': None,
'name': 'Airquality 0468fb50-0a7b-11ed-95b6-4f6e5b6c5022',
'port': 62344,
'name': 'Airquality 741b9886-0a80-11ed-95b6-4f6e5b6c5022',
'port': 35981,
'state': None},
'created': datetime.datetime(2022, 7, 23, 11, 31, 57, 23000, tzinfo=tzutc()),
'created': datetime.datetime(2022, 7, 23, 12, 10, 48, 550000, tzinfo=tzutc()),
'creator': {'affiliation': None,
'authorities': None,
'containers': None,
'databases': None,
'email': 'someone@example.com',
'email_verified': False,
'firstname': None,
'id': 2,
'identifiers': None,
'lastname': None,
'orcid': None,
'titles_after': None,
'titles_before': None,
'username': 'user'},
'deleted': None,
'description': 'Hourly measurements in Zürich, Switzerland',
'exchange': 'airquality_0e9a48ae-0a7b-11ed-95b6-4f6e5b6c5022',
'exchange': 'airquality_7c5e7aea-0a80-11ed-95b6-4f6e5b6c5022',
'id': 1,
'image': {'compiled': None,
'date_formats': [{'created_at': datetime.datetime(2022, 7, 23, 11, 29, 50, 206000, tzinfo=tzutc()),
'date_formats': [{'created_at': datetime.datetime(2022, 7, 23, 12, 8, 54, 186000, tzinfo=tzutc()),
'database_format': '%Y-%c-%d',
'example': '2022-01-30',
'has_time': False,
'id': 1,
'unix_format': 'yyyy-MM-dd'},
{'created_at': datetime.datetime(2022, 7, 23, 11, 29, 50, 215000, tzinfo=tzutc()),
{'created_at': datetime.datetime(2022, 7, 23, 12, 8, 54, 192000, tzinfo=tzutc()),
'database_format': '%d.%c.%Y',
'example': '30.01.2022',
'has_time': False,
'id': 2,
'unix_format': 'yyyy-MM-dd'},
{'created_at': datetime.datetime(2022, 7, 23, 11, 29, 50, 220000, tzinfo=tzutc()),
{'created_at': datetime.datetime(2022, 7, 23, 12, 8, 54, 195000, tzinfo=tzutc()),
'database_format': '%d.%c.%y',
'example': '30.01.22',
'has_time': False,
'id': 3,
'unix_format': 'yyyy-MM-dd'},
{'created_at': datetime.datetime(2022, 7, 23, 11, 29, 50, 224000, tzinfo=tzutc()),
{'created_at': datetime.datetime(2022, 7, 23, 12, 8, 54, 198000, tzinfo=tzutc()),
'database_format': '%c/%d/%Y',
'example': '01/30/2022',
'has_time': False,
'id': 4,
'unix_format': 'yyyy-MM-dd'},
{'created_at': datetime.datetime(2022, 7, 23, 11, 29, 50, 228000, tzinfo=tzutc()),
{'created_at': datetime.datetime(2022, 7, 23, 12, 8, 54, 201000, tzinfo=tzutc()),
'database_format': '%c/%d/%y',
'example': '01/30/22',
'has_time': False,
'id': 5,
'unix_format': 'yyyy-MM-dd'},
{'created_at': datetime.datetime(2022, 7, 23, 11, 29, 50, 231000, tzinfo=tzutc()),
{'created_at': datetime.datetime(2022, 7, 23, 12, 8, 54, 203000, tzinfo=tzutc()),
'database_format': '%Y-%c-%d %H:%i:%S.%f',
'example': '2022-01-30 13:44:25.0',
'has_time': True,
'id': 6,
'unix_format': 'yyyy-MM-dd HH:mm:ss.SSSSSS'},
{'created_at': datetime.datetime(2022, 7, 23, 11, 29, 50, 234000, tzinfo=tzutc()),
{'created_at': datetime.datetime(2022, 7, 23, 12, 8, 54, 205000, tzinfo=tzutc()),
'database_format': '%Y-%c-%d %H:%i:%S',
'example': '2022-01-30 13:44:25',
'has_time': True,
'id': 7,
'unix_format': 'yyyy-MM-dd HH:mm:ss'},
{'created_at': datetime.datetime(2022, 7, 23, 11, 29, 50, 237000, tzinfo=tzutc()),
{'created_at': datetime.datetime(2022, 7, 23, 12, 8, 54, 208000, tzinfo=tzutc()),
'database_format': '%d.%c.%Y %H:%i:%S',
'example': '30.01.2022 13:44:25',
'has_time': True,
'id': 8,
'unix_format': 'dd.MM.yyyy HH:mm:ss'}],
'default_port': 3306,
'dialect': 'org.hibernate.dialect.MariaDBDialect',
'driver_class': 'org.mariadb.jdbc.Driver',
'environment': [{'iid': 1,
'key': 'ROOT',
'type': 'PRIVILEGED_USERNAME',
'value': 'root'},
{'iid': 1,
'key': 'MARIADB_ROOT_PASSWORD',
'type': 'PRIVILEGED_PASSWORD',
'value': 'mariadb'},
{'iid': 1,
'key': 'MARIADB_USER',
'type': 'USERNAME',
'value': 'mariadb'},
{'iid': 1,
'key': 'MARIADB_PASSWORD',
'type': 'PASSWORD',
'value': 'mariadb'}],
'hash': None,
'id': 1,
'jdbc_method': 'mariadb',
'repository': 'mariadb',
'size': None,
'tag': '10.5'},
'internal_name': 'airquality_0e9a48ae-0a7b-11ed-95b6-4f6e5b6c5022',
'internal_name': 'airquality_7c5e7aea-0a80-11ed-95b6-4f6e5b6c5022',
'is_public': True,
'language': None,
'license': None,
'name': 'Airquality 0e9a48ae-0a7b-11ed-95b6-4f6e5b6c5022',
'name': 'Airquality 7c5e7aea-0a80-11ed-95b6-4f6e5b6c5022',
'publication': None,
'publisher': None,
'subjects': [],
'tables': []}
%% Cell type:markdown id: tags:
Add metadata to database
%% Cell type:code id: tags:
``` python
response = database.update({
"description": "This dataset includes daily values from 1983 to the current day, divided into annual files. This includes the maximum hourly average and the number of times the hourly average limit value for ozone was exceeded and the daily averages for sulfur dioxide (SO2), carbon monoxide (CO), nitrogen oxide (NOx), nitrogen monoxide (NO), nitrogen dioxide (NO2), particulate matter (PM10 and PM2.5). ) and particle number (PN), provided that they are of sufficient quality. The values of the completed day for the current year are updated every 30 minutes after midnight (UTC+1).",
"publisher": "Technical University of Vienna",
"license": {
"identifier": "CC0-1.0",
"uri": "https://creativecommons.org/publicdomain/zero/1.0/legalcode"
},
"language": "en",
"is_public": True,
"publication": "2022-07-19"
}, container_id, database_id)
print(response)
```
%% Output
{'container': {'created': datetime.datetime(2022, 7, 23, 11, 31, 39, 906000, tzinfo=tzutc()),
{'container': {'created': datetime.datetime(2022, 7, 23, 12, 10, 34, 731000, tzinfo=tzutc()),
'creator': {'affiliation': None,
'authorities': None,
'email': 'someone@example.com',
'email_verified': False,
'firstname': None,
'id': 2,
'lastname': None,
'orcid': None,
'titles_after': None,
'titles_before': None,
'username': 'user'},
'hash': 'afa5b9a1ef1d07f09a90f6c76c696974a194376b5d3bae48e3a442056dc30ce2',
'hash': 'a7ac6834e9c29d93a0c7049707bbb1350cbd87d9d2836fc4b19dfadc8bbe9724',
'id': 1,
'internal_name': 'fda-userdb-airquality-0468fb50-0a7b-11ed-95b6-4f6e5b6c5022',
'internal_name': 'fda-userdb-airquality-741b9886-0a80-11ed-95b6-4f6e5b6c5022',
'is_public': None,
'name': 'Airquality 0468fb50-0a7b-11ed-95b6-4f6e5b6c5022'},
'created': datetime.datetime(2022, 7, 23, 11, 31, 57, 23000, tzinfo=tzutc()),
'name': 'Airquality 741b9886-0a80-11ed-95b6-4f6e5b6c5022'},
'created': datetime.datetime(2022, 7, 23, 12, 10, 48, 550000, tzinfo=tzutc()),
'creator': {'affiliation': None,
'authorities': None,
'email': 'someone@example.com',
'email_verified': False,
'firstname': None,
'id': 2,
'lastname': None,
'orcid': None,
'titles_after': None,
'titles_before': None,
'username': 'user'},
'description': 'This dataset includes daily values from 1983 to the current '
'day, divided into annual files. This includes the maximum '
'hourly average and the number of times the hourly average '
'limit value for ozone was exceeded and the daily averages for '
'sulfur dioxide (SO2), carbon monoxide (CO), nitrogen oxide '
'(NOx), nitrogen monoxide (NO), nitrogen dioxide (NO2), '
'particulate matter (PM10 and PM2.5). ) and particle number '
'(PN), provided that they are of sufficient quality. The '
'values of the completed day for the current year are updated '
'every 30 minutes after midnight (UTC+1).',
'engine': 'mariadb:10.5',
'id': 1,
'is_public': True,
'name': 'Airquality 0e9a48ae-0a7b-11ed-95b6-4f6e5b6c5022'}
'name': 'Airquality 7c5e7aea-0a80-11ed-95b6-4f6e5b6c5022'}
%% Cell type:markdown id: tags:
Create table
%% Cell type:code id: tags:
``` python
response = table.create({
"name": "Airquality " + str(uuid.uuid1()),
"description": "Airquality in Zürich, Switzerland",
"columns": [{
"name": "Date",
"type": "date",
"dfid": 1,
"unique": False,
"primary_key": False,
"null_allowed": True,
}, {
"name": "Location",
"type": "string",
"unique": False,
"primary_key": False,
"null_allowed": True,
}, {
"name": "Parameter",
"type": "string",
"unique": False,
"primary_key": False,
"null_allowed": True,
}, {
"name": "Interval",
"type": "string",
"unique": False,
"primary_key": False,
"null_allowed": True,
}, {
"name": "Unit",
"type": "string",
"unique": False,
"primary_key": False,
"null_allowed": True,
}, {
"name": "Value",
"type": "decimal",
"unique": False,
"primary_key": False,
"null_allowed": True,
}, {
"name": "Status",
"type": "string",
"unique": False,
"primary_key": False,
"null_allowed": True,
}]
}, container_id, database_id)
table_id = response.id
print(response)
```
%% Output
{'creator': {'affiliation': None,
'authorities': None,
'email': 'someone@example.com',
'email_verified': False,
'firstname': None,
'id': 2,
'lastname': None,
'orcid': None,
'titles_after': None,
'titles_before': None,
'username': 'user'},
'id': 1,
'internal_name': 'airquality_eaf435b0-0a78-11ed-95b6-4f6e5b6c5022',
'name': 'Airquality eaf435b0-0a78-11ed-95b6-4f6e5b6c5022'}
'internal_name': 'airquality_8138a716-0a80-11ed-95b6-4f6e5b6c5022',
'name': 'Airquality 8138a716-0a80-11ed-95b6-4f6e5b6c5022'}
%% Cell type:markdown id: tags:
Inspect table
%% Cell type:code id: tags:
``` python
response = table.find_by_id(container_id, database_id, table_id)
table_internal_name = response.internal_name
table_topic = response.topic
print(response)
```
%% Output
{'columns': [{'auto_generated': True,
'check_expression': None,
'column_concept': None,
'column_type': 'number',
'date_format': None,
'decimal_digits_after': None,
'decimal_digits_before': None,
'enum_values': [],
'foreign_key': None,
'id': 1,
'internal_name': 'id',
'is_null_allowed': False,
'is_primary_key': True,
'name': 'id',
'references': None,
'unique': True},
{'auto_generated': False,
'check_expression': None,
'column_concept': None,
'column_type': 'date',
'date_format': {'created_at': datetime.datetime(2022, 7, 23, 11, 15, 16, 385000, tzinfo=tzutc()),
'date_format': {'created_at': datetime.datetime(2022, 7, 23, 12, 8, 54, 186000, tzinfo=tzutc()),
'database_format': '%Y-%c-%d',
'example': '2022-01-30',
'has_time': False,
'id': 1,
'unix_format': 'yyyy-MM-dd'},
'decimal_digits_after': None,
'decimal_digits_before': None,
'enum_values': [],
'foreign_key': None,
'id': 2,
'internal_name': 'date',
'is_null_allowed': True,
'is_primary_key': False,
'name': 'Date',
'references': None,
'unique': False},
{'auto_generated': False,
'check_expression': None,
'column_concept': None,
'column_type': 'string',
'date_format': None,
'decimal_digits_after': None,
'decimal_digits_before': None,
'enum_values': [],
'foreign_key': None,
'id': 3,
'internal_name': 'location',
'is_null_allowed': True,
'is_primary_key': False,
'name': 'Location',
'references': None,
'unique': False},
{'auto_generated': False,
'check_expression': None,
'column_concept': None,
'column_type': 'string',
'date_format': None,
'decimal_digits_after': None,
'decimal_digits_before': None,
'enum_values': [],
'foreign_key': None,
'id': 4,
'internal_name': 'parameter',
'is_null_allowed': True,
'is_primary_key': False,
'name': 'Parameter',
'references': None,
'unique': False},
{'auto_generated': False,
'check_expression': None,
'column_concept': None,
'column_type': 'string',
'date_format': None,
'decimal_digits_after': None,
'decimal_digits_before': None,
'enum_values': [],
'foreign_key': None,
'id': 5,
'internal_name': 'interval',
'is_null_allowed': True,
'is_primary_key': False,
'name': 'Interval',
'references': None,
'unique': False},
{'auto_generated': False,
'check_expression': None,
'column_concept': None,
'column_type': 'string',
'date_format': None,
'decimal_digits_after': None,
'decimal_digits_before': None,
'enum_values': [],
'foreign_key': None,
'id': 6,
'internal_name': 'unit',
'is_null_allowed': True,
'is_primary_key': False,
'name': 'Unit',
'references': None,
'unique': False},
{'auto_generated': False,
'check_expression': None,
'column_concept': None,
'column_type': 'decimal',
'date_format': None,
'decimal_digits_after': None,
'decimal_digits_before': None,
'enum_values': [],
'foreign_key': None,
'id': 7,
'internal_name': 'value',
'is_null_allowed': True,
'is_primary_key': False,
'name': 'Value',
'references': None,
'unique': False},
{'auto_generated': False,
'check_expression': None,
'column_concept': None,
'column_type': 'string',
'date_format': None,
'decimal_digits_after': None,
'decimal_digits_before': None,
'enum_values': [],
'foreign_key': None,
'id': 8,
'internal_name': 'status',
'is_null_allowed': True,
'is_primary_key': False,
'name': 'Status',
'references': None,
'unique': False}],
'created': datetime.datetime(2022, 7, 23, 11, 16, 38, 656000, tzinfo=tzutc()),
'created': datetime.datetime(2022, 7, 23, 12, 10, 56, 764000, tzinfo=tzutc()),
'description': 'Airquality in Zürich, Switzerland',
'id': 1,
'internal_name': 'airquality_eaf435b0-0a78-11ed-95b6-4f6e5b6c5022',
'name': 'Airquality eaf435b0-0a78-11ed-95b6-4f6e5b6c5022',
'topic': 'airquality_eaf435b0-0a78-11ed-95b6-4f6e5b6c5022'}
'internal_name': 'airquality_8138a716-0a80-11ed-95b6-4f6e5b6c5022',
'name': 'Airquality 8138a716-0a80-11ed-95b6-4f6e5b6c5022',
'topic': 'airquality_8138a716-0a80-11ed-95b6-4f6e5b6c5022'}
%% Cell type:markdown id: tags:
Import .csv
%% Cell type:code id: tags:
``` python
shutil.copyfile(os.getcwd() + "/resources/ugz_ogd_air_h1_2021.csv", "/tmp/ugz_ogd_air_h1_2021.csv")
response = data.import_csv({
"location": "/tmp/ugz_ogd_air_h1_2021.csv",
"separator": ",",
"quote": "\"",
"skip_lines": 1
}, container_id, database_id, table_id)
print(response)
```
%% Output
210192
%% Cell type:markdown id: tags:
Create subset
%% Cell type:code id: tags:
``` python
response = query.execute({
"statement": "select `date`, `location`, `parameter`, `interval`, `unit`, `value`, `status` from `" + table_internal_name + "` where `location` = 'Schimmelstrasse'"
}, container_id, database_id, page=0, size=3)
query_id = response.id
print(response)
```
%% Output
{'id': 1,
'result': [{'Date': '2021-01-01T00:00:00Z',
'Interval': 'h1',
'Location': 'Schimmelstrasse',
'Parameter': 'NOx',
'Status': 'tentative',
'Unit': 'ppb',
'Value': 41.66},
{'Date': '2021-01-01T00:00:00Z',
'Interval': 'h1',
'Location': 'Schimmelstrasse',
'Parameter': 'NO',
'Status': 'tentative',
'Unit': 'µg/m3',
'Value': 21.64},
{'Date': '2021-01-01T00:00:00Z',
'Interval': 'h1',
'Location': 'Schimmelstrasse',
'Parameter': 'NO2',
'Status': 'tentative',
'Unit': 'µg/m3',
'Value': 46.49}],
'result_number': 52548}
%% Cell type:markdown id: tags:
Create invalid subset
%% Cell type:code id: tags:
``` python
response = query.execute({
"statement": "select `date`, `location`, `parameter`, `interval`, `unit`, `value`, `status` from `" + table_internal_name + "` where `foo` = 'bar'"
}, container_id, database_id, page=0, size=3)
print(response)
```
%% Output
---------------------------------------------------------------------------
ApiException Traceback (most recent call last)
Input In [98], in <cell line: 1>()
----> 1 response = query.execute({
2 "statement": "select `date`, `location`, `parameter`, `interval`, `unit`, `value`, `status` from `" + table_internal_name + "` where `foo` = 'bar'"
3 }, container_id, database_id, page=0, size=3)
4 print(response)
File ~/Projects/fda-services/.jupyter/api_query/api/query_endpoint_api.py:57, in QueryEndpointApi.execute(self, body, id, database_id, **kwargs)
55 return self.execute_with_http_info(body, id, database_id, **kwargs) # noqa: E501
56 else:
---> 57 (data) = self.execute_with_http_info(body, id, database_id, **kwargs) # noqa: E501
58 return data
File ~/Projects/fda-services/.jupyter/api_query/api/query_endpoint_api.py:140, in QueryEndpointApi.execute_with_http_info(self, body, id, database_id, **kwargs)
137 # Authentication setting
138 auth_settings = ['bearerAuth'] # noqa: E501
--> 140 return self.api_client.call_api(
141 '/api/container/{id}/database/{databaseId}/query', 'PUT',
142 path_params,
143 query_params,
144 header_params,
145 body=body_params,
146 post_params=form_params,
147 files=local_var_files,
148 response_type='QueryResultDto', # noqa: E501
149 auth_settings=auth_settings,
150 async_req=params.get('async_req'),
151 _return_http_data_only=params.get('_return_http_data_only'),
152 _preload_content=params.get('_preload_content', True),
153 _request_timeout=params.get('_request_timeout'),
154 collection_formats=collection_formats)
File ~/Projects/fda-services/.jupyter/api_query/api_client.py:316, in ApiClient.call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, async_req, _return_http_data_only, collection_formats, _preload_content, _request_timeout)
279 """Makes the HTTP request (synchronous) and returns deserialized data.
280
281 To make an async request, set the async_req parameter.
(...)
313 then the method will return the response directly.
314 """
315 if not async_req:
--> 316 return self.__call_api(resource_path, method,
317 path_params, query_params, header_params,
318 body, post_params, files,
319 response_type, auth_settings,
320 _return_http_data_only, collection_formats,
321 _preload_content, _request_timeout)
322 else:
323 thread = self.pool.apply_async(self.__call_api, (resource_path,
324 method, path_params, query_params,
325 header_params, body,
(...)
329 collection_formats,
330 _preload_content, _request_timeout))
File ~/Projects/fda-services/.jupyter/api_query/api_client.py:148, in ApiClient.__call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout)
145 url = self.configuration.host + resource_path
147 # perform request and return response
--> 148 response_data = self.request(
149 method, url, query_params=query_params, headers=header_params,
150 post_params=post_params, body=body,
151 _preload_content=_preload_content,
152 _request_timeout=_request_timeout)
154 self.last_response = response_data
156 return_data = response_data
File ~/Projects/fda-services/.jupyter/api_query/api_client.py:366, in ApiClient.request(self, method, url, query_params, headers, post_params, body, _preload_content, _request_timeout)
358 return self.rest_client.POST(url,
359 query_params=query_params,
360 headers=headers,
(...)
363 _request_timeout=_request_timeout,
364 body=body)
365 elif method == "PUT":
--> 366 return self.rest_client.PUT(url,
367 query_params=query_params,
368 headers=headers,
369 post_params=post_params,
370 _preload_content=_preload_content,
371 _request_timeout=_request_timeout,
372 body=body)
373 elif method == "PATCH":
374 return self.rest_client.PATCH(url,
375 query_params=query_params,
376 headers=headers,
(...)
379 _request_timeout=_request_timeout,
380 body=body)
File ~/Projects/fda-services/.jupyter/api_query/rest.py:273, in RESTClientObject.PUT(self, url, headers, query_params, post_params, body, _preload_content, _request_timeout)
271 def PUT(self, url, headers=None, query_params=None, post_params=None,
272 body=None, _preload_content=True, _request_timeout=None):
--> 273 return self.request("PUT", url,
274 headers=headers,
275 query_params=query_params,
276 post_params=post_params,
277 _preload_content=_preload_content,
278 _request_timeout=_request_timeout,
279 body=body)
File ~/Projects/fda-services/.jupyter/api_query/rest.py:222, in RESTClientObject.request(self, method, url, query_params, headers, body, post_params, _preload_content, _request_timeout)
219 logger.debug("response body: %s", r.data)
221 if not 200 <= r.status <= 299:
--> 222 raise ApiException(http_resp=r)
224 return r
ApiException: (400)
Reason: Bad Request
HTTP response headers: HTTPHeaderDict({'transfer-encoding': 'chunked', 'Vary': 'Origin, Access-Control-Request-Method, Access-Control-Request-Headers', 'X-Content-Type-Options': 'nosniff', 'X-XSS-Protection': '1; mode=block', 'Cache-Control': 'no-cache, no-store, max-age=0, must-revalidate', 'Pragma': 'no-cache', 'Expires': '0', 'X-Frame-Options': 'DENY', 'Content-Type': 'application/json', 'Date': 'Sat, 23 Jul 2022 11:16:44 GMT'})
HTTP response body: b'{"status":"BAD_REQUEST","message":"Query not valid for this database","code":"error.query.malformed"}'
%% Cell type:code id: tags:
``` python
```
......
fda-table-service/pom.xml
+ 6
− 0
Edit View file @ 10a2c221
......@@ -75,6 +75,12 @@
<artifactId>fda-metadata-db-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>at.tuwien</groupId>
<artifactId>fda-metadata-db-querystore</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-c3p0</artifactId>
......
fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java
+ 2
− 2
Edit View file @ 10a2c221
......@@ -68,7 +68,7 @@ public class TableEndpoint extends AbstractEndpoint {
@NotNull @Valid @RequestBody TableCreateDto createDto,
Principal principal)
throws ImageNotSupportedException, DatabaseNotFoundException, TableMalformedException, AmqpException,
TableNameExistsException, ContainerNotFoundException, UserNotFoundException {
TableNameExistsException, ContainerNotFoundException, UserNotFoundException, DatabaseConnectionException {
if (!hasDatabasePermission(containerId, databaseId, "TABLE_CREATE", principal)) {
log.error("Missing table create permission");
throw new NotAllowedException("Missing table create permission");
......@@ -122,7 +122,7 @@ public class TableEndpoint extends AbstractEndpoint {
@NotNull @PathVariable("tableId") Long tableId,
Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
DataProcessingException, ContainerNotFoundException, TableMalformedException {
DataProcessingException, ContainerNotFoundException, TableMalformedException, DatabaseConnectionException {
if (!hasTablePermission(containerId, databaseId, tableId, "TABLE_DELETE", principal)) {
log.error("Missing table delete permission");
throw new NotAllowedException("Missing table delete permission");
......
fda-table-service/rest-service/src/main/resources/mariadb_hibernate.cfg.xml
+ 3
− 0
Edit View file @ 10a2c221
......@@ -13,5 +13,8 @@
<property name="show_sql">true</property>
<property name="format_sql">true</property>
<property name="hbm2ddl.auto">update</property>
<mapping class="at.tuwien.querystore.Query" />
<mapping class="at.tuwien.querystore.Table" />
<mapping class="at.tuwien.querystore.Column" />
</session-factory>
</hibernate-configuration>
fda-table-service/rest-service/src/test/java/at/tuwien/CsvUnitTest.java deleted 100644 → 0
+ 0
− 53
View file @ 310c1091
package at.tuwien;
import at.tuwien.api.database.table.columns.ColumnCreateDto;
import at.tuwien.api.database.table.columns.ColumnTypeDto;
public abstract class CsvUnitTest {
public final static ColumnCreateDto[] COLUMNS_CSV02 = new ColumnCreateDto[]{
ColumnCreateDto.builder()
.type(ColumnTypeDto.NUMBER)
.name("id")
.nullAllowed(false)
.primaryKey(true)
.unique(true)
.build(),
ColumnCreateDto.builder()
.type(ColumnTypeDto.STRING)
.name("nouniquestr")
.nullAllowed(false)
.primaryKey(false)
.unique(false)
.build(),
ColumnCreateDto.builder()
.type(ColumnTypeDto.ENUM)
.name("method")
.nullAllowed(false)
.primaryKey(false)
.unique(false)
.enumValues(new String[] {"mk", "zf", "ac", "em"})
.build(),
ColumnCreateDto.builder()
.type(ColumnTypeDto.STRING)
.name("company")
.nullAllowed(false)
.primaryKey(false)
.unique(false)
.build(),
ColumnCreateDto.builder()
.type(ColumnTypeDto.NUMBER)
.name("measurements")
.nullAllowed(false)
.primaryKey(false)
.unique(false)
.build(),
ColumnCreateDto.builder()
.type(ColumnTypeDto.NUMBER)
.name("trialn")
.nullAllowed(false)
.primaryKey(false)
.unique(false)
.build(),};
}
fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointIntegrationTest.java deleted 100644 → 0
+ 0
− 136
View file @ 310c1091
package at.tuwien.endpoint;
import at.tuwien.BaseUnitTest;
import at.tuwien.api.database.table.TableBriefDto;
import at.tuwien.api.database.table.TableCreateDto;
import at.tuwien.config.DockerConfig;
import at.tuwien.config.ReadyConfig;
import at.tuwien.endpoints.TableEndpoint;
import at.tuwien.exception.*;
import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.jpa.ImageRepository;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.exception.NotModifiedException;
import com.github.dockerjava.api.model.Network;
import com.rabbitmq.client.Channel;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.auth.BasicUserPrincipal;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.transaction.annotation.Transactional;
import java.security.Principal;
import java.util.Arrays;
import static at.tuwien.config.DockerConfig.dockerClient;
import static at.tuwien.config.DockerConfig.hostConfig;
import static org.junit.jupiter.api.Assertions.assertEquals;
@Slf4j
@SpringBootTest
@ExtendWith(SpringExtension.class)
public class TableEndpointIntegrationTest extends BaseUnitTest {
@MockBean
private Channel channel;
@MockBean
private ReadyConfig readyConfig;
@Autowired
private TableEndpoint tableEndpoint;
@Autowired
private ImageRepository imageRepository;
@Autowired
private DatabaseRepository databaseRepository;
@BeforeEach
@Transactional
public void beforeEach() {
afterEach();
/* create network */
dockerClient.createNetworkCmd()
.withName("fda-userdb")
.withIpam(new Network.Ipam()
.withConfig(new Network.Ipam.Config()
.withSubnet("172.28.0.0/16")))
.withEnableIpv6(false)
.exec();
/* create container */
final CreateContainerResponse response = dockerClient.createContainerCmd(IMAGE_1_REPOSITORY + ":" + IMAGE_1_TAG)
.withHostConfig(hostConfig.withNetworkMode("fda-userdb"))
.withName(CONTAINER_3_INTERNALNAME)
.withIpv4Address(CONTAINER_3_IP)
.withHostName(CONTAINER_3_INTERNALNAME)
.withEnv("MARIADB_USER=mariadb", "MARIADB_PASSWORD=mariadb", "MARIADB_ROOT_PASSWORD=mariadb",
"MARIADB_DATABASE=traffic")
.exec();
CONTAINER_3.setHash(response.getId());
/* repository */
imageRepository.save(IMAGE_1);
databaseRepository.save(DATABASE_1);
databaseRepository.save(DATABASE_2);
databaseRepository.save(DATABASE_3);
}
@AfterEach
public void afterEach() {
/* stop containers and remove them */
dockerClient.listContainersCmd()
.withShowAll(true)
.exec()
.forEach(container -> {
log.info("Delete container {}", Arrays.asList(container.getNames()));
try {
dockerClient.stopContainerCmd(container.getId()).exec();
} catch (NotModifiedException e) {
// ignore
}
dockerClient.removeContainerCmd(container.getId()).exec();
});
/* remove networks */
dockerClient.listNetworksCmd()
.exec()
.stream()
.filter(n -> n.getName().startsWith("fda"))
.forEach(network -> {
log.info("Delete network {}", network.getName());
dockerClient.removeNetworkCmd(network.getId()).exec();
});
}
@Test
public void create_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException,
TableMalformedException, AmqpException, TableNameExistsException, InterruptedException,
ContainerNotFoundException,
UserNotFoundException {
final TableCreateDto request = TableCreateDto.builder()
.name(TABLE_3_NAME)
.description(TABLE_3_DESCRIPTION)
.columns(COLUMNS_CSV_CH)
.build();
final Principal principal = new BasicUserPrincipal(USER_1_USERNAME);
/* mock */
DockerConfig.startContainer(CONTAINER_3);
/* test */
final ResponseEntity<TableBriefDto> response = tableEndpoint.create(CONTAINER_3_ID, DATABASE_3_ID, request,
principal);
assertEquals(HttpStatus.CREATED, response.getStatusCode());
}
}
fda-table-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java deleted 100644 → 0
+ 0
− 186
View file @ 310c1091
package at.tuwien.endpoint;
import at.tuwien.BaseUnitTest;
import at.tuwien.api.database.table.TableBriefDto;
import at.tuwien.api.database.table.TableCreateDto;
import at.tuwien.api.database.table.TableDto;
import at.tuwien.config.ReadyConfig;
import at.tuwien.endpoints.TableEndpoint;
import at.tuwien.exception.*;
import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.jpa.TableRepository;
import at.tuwien.service.MessageQueueService;
import at.tuwien.service.impl.TableServiceImpl;
import com.rabbitmq.client.Channel;
import org.apache.http.auth.BasicUserPrincipal;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.io.IOException;
import java.security.Principal;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.*;
@SpringBootTest
@ExtendWith(SpringExtension.class)
public class TableEndpointUnitTest extends BaseUnitTest {
@MockBean
private Channel channel;
@MockBean
private ReadyConfig readyConfig;
@MockBean
private TableServiceImpl tableService;
@MockBean
private TableRepository tableRepository;
@MockBean
private DatabaseRepository databaseRepository;
@MockBean
private MessageQueueService messageQueueService;
@Autowired
private TableEndpoint tableEndpoint;
// @Test
// public void findAll_succeeds() throws DatabaseNotFoundException {
//
// /* mock */
// when(tableRepository.findByDatabase(DATABASE_1))
// .thenReturn(List.of(TABLE_1));
// when(tableService.findAll(CONTAINER_1_ID, DATABASE_1_ID))
// .thenReturn(List.of(TABLE_1));
//
// /* test */
// final ResponseEntity<List<TableBriefDto>> response = tableEndpoint.findAll(CONTAINER_1_ID, DATABASE_1_ID);
// assertEquals(HttpStatus.OK, response.getStatusCode());
// assertEquals(1, Objects.requireNonNull(response.getBody()).size());
// }
//
// @Test
// public void create_succeeds() throws DatabaseNotFoundException, ImageNotSupportedException,
// TableNotFoundException, TableMalformedException, AmqpException, TableNameExistsException,
// ContainerNotFoundException, UserNotFoundException {
// final TableCreateDto request = TableCreateDto.builder()
// .name(TABLE_1_NAME)
// .description(TABLE_1_DESCRIPTION)
// .columns(COLUMNS_CSV01)
// .build();
// final Principal principal = new BasicUserPrincipal(USER_1_USERNAME);
//
// /* mock */
// when(tableRepository.findById(TABLE_1_ID))
// .thenReturn(Optional.of(TABLE_1));
// when(tableService.findById(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID))
// .thenReturn(TABLE_1);
// doNothing()
// .when(messageQueueService)
// .create(TABLE_1);
//
// /* test */
// final ResponseEntity<TableBriefDto> response = tableEndpoint.create(CONTAINER_1_ID, DATABASE_1_ID, request,
// principal);
// assertEquals(HttpStatus.CREATED, response.getStatusCode());
// }
//
// @Test
// public void create_databaseNotFound_fails() throws DatabaseNotFoundException, ImageNotSupportedException,
// TableMalformedException, TableNameExistsException, ContainerNotFoundException, UserNotFoundException {
// final TableCreateDto request = TableCreateDto.builder()
// .name(TABLE_1_NAME)
// .description(TABLE_1_DESCRIPTION)
// .columns(COLUMNS_CSV01)
// .build();
// final Principal principal = new BasicUserPrincipal(USER_1_USERNAME);
//
// /* mock */
// when(tableService.createTable(CONTAINER_1_ID, DATABASE_1_ID, request, principal))
// .thenAnswer(invocation -> {
// throw new DatabaseNotFoundException("no db");
// });
//
// /* test */
// assertThrows(DatabaseNotFoundException.class, () -> {
// tableEndpoint.create(CONTAINER_1_ID, DATABASE_1_ID, request, principal);
// });
// }
//
// @Test
// public void findById_succeeds() throws TableNotFoundException, DatabaseNotFoundException,
// ContainerNotFoundException {
//
// /* mock */
// when(tableService.findById(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID))
// .thenReturn(TABLE_1);
//
// /* test */
// final ResponseEntity<TableDto> response = tableEndpoint.findById(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID);
// assertEquals(HttpStatus.OK, response.getStatusCode());
// assertEquals(TABLE_1_ID, Objects.requireNonNull(response.getBody()).getId());
// assertEquals(TABLE_1_NAME, Objects.requireNonNull(response.getBody()).getName());
// }
//
// @Test
// public void findById_notFound_fails() throws TableNotFoundException, DatabaseNotFoundException,
// ContainerNotFoundException {
//
// /* mock */
// when(tableRepository.findById(TABLE_1_ID))
// .thenReturn(Optional.empty());
// doThrow(TableNotFoundException.class)
// .when(tableService)
// .findById(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID);
//
// /* test */
// assertThrows(TableNotFoundException.class, () -> {
// tableEndpoint.findById(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID);
// });
// }
//
// @Test
// public void delete_notFound_fails() throws TableNotFoundException, DatabaseNotFoundException,
// ImageNotSupportedException, ContainerNotFoundException {
//
// /* mock */
// doThrow(TableNotFoundException.class)
// .when(tableService)
// .deleteTable(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID);
//
// /* test */
// assertThrows(TableNotFoundException.class, () -> {
// tableEndpoint.delete(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID);
// });
// }
//
// @Test
// public void delete_succeeds() throws TableNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
// DataProcessingException, ContainerNotFoundException {
//
// /* test */
// tableEndpoint.delete(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID);
// }
//
// @Test
// public void update_fails() {
//
// /* test */
// final ResponseEntity<TableBriefDto> response = tableEndpoint.update(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID);
// assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, response.getStatusCode());
// }
}
fda-table-service/rest-service/src/test/java/at/tuwien/service/HibernateConnectorUnitTest.java deleted 100644 → 0
+ 0
− 68
View file @ 310c1091
package at.tuwien.service;
import at.tuwien.BaseUnitTest;
import at.tuwien.api.database.table.TableCsvDto;
import at.tuwien.config.DockerConfig;
import at.tuwien.config.ReadyConfig;
import at.tuwien.exception.*;
import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.jpa.TableRepository;
import at.tuwien.service.impl.HibernateConnector;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.exception.NotModifiedException;
import com.github.dockerjava.api.model.Bind;
import com.github.dockerjava.api.model.Network;
import com.rabbitmq.client.Channel;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.io.File;
import java.io.IOException;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static at.tuwien.config.DockerConfig.dockerClient;
import static at.tuwien.config.DockerConfig.hostConfig;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.when;
@SpringBootTest
@ExtendWith(SpringExtension.class)
@Log4j2
public class HibernateConnectorUnitTest extends BaseUnitTest {
@MockBean
private Channel channel;
@MockBean
private ReadyConfig readyConfig;
@Test
public void isReserved_succeeds() throws IOException {
final String request = "TIMESTAMP";
/* test */
assertTrue(HibernateConnector.isReserved(request));
}
@Test
public void isReserved_fails() throws IOException {
final String request = "foobar";
/* test */
assertFalse(HibernateConnector.isReserved(request));
}
}
fda-table-service/rest-service/src/test/java/at/tuwien/service/TableServiceUnitTest.java deleted 100644 → 0
+ 0
− 125
View file @ 310c1091
package at.tuwien.service;
import at.tuwien.BaseUnitTest;
import at.tuwien.config.ReadyConfig;
import at.tuwien.entities.database.table.Table;
import at.tuwien.exception.*;
import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.jpa.TableRepository;
import at.tuwien.service.impl.TableServiceImpl;
import com.rabbitmq.client.Channel;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import java.util.List;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.*;
@SpringBootTest
@ExtendWith(SpringExtension.class)
public class TableServiceUnitTest extends BaseUnitTest {
@MockBean
private Channel channel;
@MockBean
private ReadyConfig readyConfig;
@Autowired
private TableServiceImpl tableService;
@MockBean
private DatabaseRepository databaseRepository;
@MockBean
private TableRepository tableRepository;
@BeforeAll
public static void beforeAll() {
TABLE_1.setDatabase(DATABASE_1);
}
// @Test
// public void findAll_succeeds() throws DatabaseNotFoundException {
// when(databaseRepository.findById(DATABASE_1_ID))
// .thenReturn(Optional.of(DATABASE_1));
// when(tableRepository.findByDatabase(DATABASE_1))
// .thenReturn(List.of(TABLE_1));
//
// /* test */
// final List<Table> response = tableService.findAll(CONTAINER_1_ID, DATABASE_1_ID);
// assertEquals(1, response.size());
// assertEquals(TABLE_1_ID, response.get(0).getId());
// }
//
// @Test
// public void findAll_notFound_fails() {
// when(databaseRepository.findById(DATABASE_1_ID))
// .thenReturn(Optional.empty());
//
// /* test */
// assertThrows(DatabaseNotFoundException.class, () -> {
// tableService.findAll(CONTAINER_1_ID, DATABASE_1_ID);
// });
// }
//
// @Test
// public void delete_notFound_fails() {
// when(databaseRepository.findById(DATABASE_1_ID))
// .thenReturn(Optional.empty());
//
// /* test */
// assertThrows(DatabaseNotFoundException.class, () -> {
// tableService.deleteTable(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID);
// });
// }
//
// @Test
// public void delete_noSql_fails() {
// when(databaseRepository.findById(DATABASE_1_ID))
// .thenReturn(Optional.of(DATABASE_1));
// when(tableRepository.findByDatabaseAndId(DATABASE_1, TABLE_1_ID))
// .thenReturn(Optional.empty());
//
// /* test */
// assertThrows(TableNotFoundException.class, () -> {
// tableService.deleteTable(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID);
// });
// }
//
// @Test
// public void findById_succeeds() throws TableNotFoundException, DatabaseNotFoundException,
// ContainerNotFoundException {
// when(databaseRepository.findById(DATABASE_1_ID))
// .thenReturn(Optional.of(DATABASE_1));
// when(tableRepository.findByDatabaseAndId(DATABASE_1, TABLE_1_ID))
// .thenReturn(Optional.of(TABLE_1));
//
// /* test */
// final Table response = tableService.findById(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID);
// assertEquals(TABLE_1_ID, response.getId());
// assertEquals(TABLE_1_NAME, response.getName());
// }
//
// @Test
// public void findById_noTable_fails() {
// when(databaseRepository.findById(DATABASE_1_ID))
// .thenReturn(Optional.of(DATABASE_1));
// when(tableRepository.findByDatabaseAndId(DATABASE_1, 9999L))
// .thenReturn(Optional.empty());
//
// /* test */
// assertThrows(TableNotFoundException.class, () -> {
// tableService.findById(CONTAINER_1_ID, DATABASE_1_ID, 9999L);
// });
// }
}
fda-table-service/services/src/main/java/at/tuwien/service/TableService.java
+ 2
− 2
Edit View file @ 10a2c221
......@@ -33,7 +33,7 @@ public interface TableService {
*/
void deleteTable(Long containerId, Long databaseId, Long tableId, Principal principal)
throws TableNotFoundException, DatabaseNotFoundException,
ImageNotSupportedException, DataProcessingException, ContainerNotFoundException, TableMalformedException;
ImageNotSupportedException, DataProcessingException, ContainerNotFoundException, TableMalformedException, DatabaseConnectionException;
/**
* Find a table by database-table id pair
......@@ -66,5 +66,5 @@ public interface TableService {
*/
Table createTable(Long containerId, Long databaseId, TableCreateDto createDto, Principal principal)
throws ImageNotSupportedException, DatabaseNotFoundException, TableMalformedException,
TableNameExistsException, ContainerNotFoundException, UserNotFoundException;
TableNameExistsException, ContainerNotFoundException, UserNotFoundException, DatabaseConnectionException;
}
fda-table-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java
+ 45
− 55
Edit View file @ 10a2c221
......@@ -5,89 +5,79 @@ import at.tuwien.entities.container.image.ContainerImage;
import at.tuwien.entities.container.image.ContainerImageEnvironmentItem;
import at.tuwien.entities.container.image.ContainerImageEnvironmentItemType;
import at.tuwien.entities.database.Database;
import at.tuwien.exception.DatabaseConnectionException;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import lombok.extern.log4j.Log4j2;
import org.apache.commons.io.IOUtils;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.hibernate.boot.Metadata;
import org.hibernate.boot.MetadataSources;
import org.hibernate.boot.registry.StandardServiceRegistryBuilder;
import org.hibernate.cfg.Configuration;
import org.hibernate.query.NativeQuery;
import org.hibernate.service.ServiceRegistry;
import org.springframework.core.io.ClassPathResource;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.persistence.PersistenceException;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.stream.Collectors;
@Log4j2
@Service
public abstract class HibernateConnector {
protected static Session getCurrentSession(ContainerImage image, Container container, Database database) {
final String url = "jdbc:" + image.getJdbcMethod() + "://" + container.getInternalName() + "/" + database.getInternalName();
protected static Connection getConnection(ContainerImage image, Container container, Database database) throws DatabaseConnectionException {
final ComboPooledDataSource dataSource = new ComboPooledDataSource();
final String url = "jdbc:" + image.getJdbcMethod() + "://" + container.getInternalName() + "/" + (database != null ? database.getInternalName() : "");
dataSource.setJdbcUrl(url);
final String username = image.getEnvironment()
.stream()
.filter(e -> e.getType().equals(ContainerImageEnvironmentItemType.PRIVILEGED_USERNAME))
.map(ContainerImageEnvironmentItem::getValue)
.collect(Collectors.toList())
.get(0);
dataSource.setUser(username);
final String password = image.getEnvironment()
.stream()
.filter(e -> e.getType().equals(ContainerImageEnvironmentItemType.PRIVILEGED_PASSWORD))
.map(ContainerImageEnvironmentItem::getValue)
.collect(Collectors.toList())
.get(0);
final Configuration config = new Configuration();
config.configure("mariadb_hibernate.cfg.xml");
config.setProperty("hibernate.connection.url", url);
config.setProperty("hibernate.connection.username", username);
config.setProperty("hibernate.connection.password", password);
config.setProperty("hibernate.connection.driver_class", image.getDriverClass());
config.setProperty("hibernate.dialect", image.getDialect());
final SessionFactory sessionFactory = config.buildSessionFactory();
Session session = sessionFactory.getCurrentSession();
if (!session.isOpen()) {
log.warn("Session is closed, opening...");
session = sessionFactory.openSession();
dataSource.setPassword(password);
dataSource.setInitialPoolSize(5);
dataSource.setMinPoolSize(5);
dataSource.setAcquireIncrement(5);
dataSource.setMaxPoolSize(20);
dataSource.setMaxStatements(100);
final Connection connection;
try {
connection = dataSource.getConnection();
} catch (SQLException e) {
log.error("Failed to connect to the database");
log.debug("failed to connect to the database {}", database);
throw new DatabaseConnectionException("Failed to connect to the database");
}
return session;
return connection;
}
protected static Long activeConnection(Session session) {
final NativeQuery<?> nativeQuery = session.createSQLQuery("SHOW STATUS LIKE 'threads_connected'");
final List<?> result;
protected static Long activeConnection(Connection connection) throws DatabaseConnectionException {
final ResultSet resultSet = execute(connection, "SHOW STATUS LIKE 'threads_connected'");
try {
result = nativeQuery.getResultList();
} catch (PersistenceException e) {
log.error("Failed to collect number of used connections");
/* ignore */
return null;
if (resultSet.next()) {
return resultSet.getLong(2);
}
final Object[] row = (Object[]) result.get(0);
log.debug("current number of connections: {}", Long.parseLong(String.valueOf(row[1])));
return Long.parseLong(String.valueOf(row[1]));
} catch (SQLException e) {
log.error("Failed to determine active connections");
throw new DatabaseConnectionException("Failed to determine active connections", e);
}
/**
* Checks if the word is in the reserved word csv (i.e. an SQL keyword), solves issue 106
*
* @param word The word
* @return True if it is reserved word
*/
public static Boolean isReserved(String word) throws IOException {
final InputStream stream = new ClassPathResource("mariadb/reserved.csv").getInputStream();
final List<String> reserved = IOUtils.readLines(stream, "UTF-8");
return reserved.contains(word.toUpperCase());
log.error("Failed to determine active connections");
throw new DatabaseConnectionException("Failed to determine active connections");
}
protected static ResultSet execute(Connection connection, String statement) throws DatabaseConnectionException {
final PreparedStatement preparedStatement;
try {
preparedStatement = connection.prepareStatement(statement);
return preparedStatement.executeQuery();
} catch (SQLException e) {
log.error("Failed to execute statement");
log.debug("failed to execute statement {}", statement);
throw new DatabaseConnectionException("Failed to execute statement", e);
}
}
}
fda-table-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
+ 12
− 51
Edit View file @ 10a2c221
......@@ -24,6 +24,7 @@ import javax.persistence.PersistenceContext;
import javax.persistence.PersistenceException;
import javax.persistence.PersistenceUnit;
import java.security.Principal;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.*;
import java.util.stream.Collectors;
......@@ -57,24 +58,16 @@ public class TableServiceImpl extends HibernateConnector implements TableService
@Override
@Transactional
public void deleteTable(Long containerId, Long databaseId, Long tableId, Principal principal)
throws TableNotFoundException, DatabaseNotFoundException, ImageNotSupportedException, TableMalformedException {
throws TableNotFoundException, DatabaseNotFoundException, ImageNotSupportedException,
TableMalformedException, DatabaseConnectionException {
/* find */
final Database database = databaseService.findPublicOrMineById(containerId, databaseId, principal);
final Table table = findById(containerId, databaseId, tableId, principal);
/* run query */
final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
final Transaction transaction = session.beginTransaction();
try {
session.createSQLQuery(tableMapper.tableToDropTableRawQuery(table));
activeConnection(session);
transaction.commit();
final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
execute(connection, tableMapper.tableToDropTableRawQuery(table));
log.info("Deleted table with id {}", table.getId());
log.debug("deleted table {}", table);
} catch (PersistenceException e) {
log.error("Failed to drop table with id {}", tableId);
log.debug("failed to drop table {}", table);
throw new TableMalformedException("Failed to drop table");
}
}
@Override
......@@ -94,7 +87,7 @@ public class TableServiceImpl extends HibernateConnector implements TableService
@Transactional
public Table createTable(Long containerId, Long databaseId, TableCreateDto createDto, Principal principal)
throws ImageNotSupportedException, DatabaseNotFoundException, TableMalformedException,
TableNameExistsException, UserNotFoundException {
TableNameExistsException, UserNotFoundException, DatabaseConnectionException {
/* find */
final Database database = databaseService.findPublicOrMineById(containerId, databaseId, principal);
final Optional<Table> optional = tableRepository.findByDatabaseAndInternalName(database,
......@@ -105,33 +98,14 @@ public class TableServiceImpl extends HibernateConnector implements TableService
throw new TableNameExistsException("Table name exists");
}
/* run query */
final Session session = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
final Transaction transaction = session.beginTransaction();
final Connection connection = getConnection(database.getContainer().getImage(), database.getContainer(), database);
final CreateTableRawQuery query = tableMapper.tableToCreateTableRawQuery(database, createDto);
if (query.getGenerated()) {
/* in case the id column needs to be generated, we need to generate the sequence too */
try {
session.createSQLQuery(tableMapper.tableToCreateSequenceRawQuery(database, createDto))
.executeUpdate();
} catch (PersistenceException e) {
log.error("Table sequence exists, but table does not. Create an issue for this.");
session.close();
throw new TableNameExistsException("Sequence exists", e);
}
execute(connection, tableMapper.tableToCreateSequenceRawQuery(database, createDto));
log.debug("created id sequence");
}
try {
session.createSQLQuery(query.getQuery())
.executeUpdate();
activeConnection(session);
transaction.commit();
} catch (PersistenceException e) {
log.error("Failed to create table");
log.debug("failed to create table: {}", e.getMessage());
session.close();
throw new TableMalformedException("Failed to create table", e);
}
session.close();
execute(connection, query.getQuery());
int[] idx = {0};
/* map table */
final Table tmp = tableMapper.tableCreateDtoToTable(createDto);
......@@ -155,20 +129,7 @@ public class TableServiceImpl extends HibernateConnector implements TableService
column.setOrdinalPosition(idx[0]++);
});
/* create history view */
final Session session2 = getCurrentSession(database.getContainer().getImage(), database.getContainer(), database);
try {
final Transaction transaction2 = session2.beginTransaction();
session2.createSQLQuery(tableMapper.tableToCreateHistoryViewRawQuery(entity))
.executeUpdate();
activeConnection(session2);
transaction2.commit();
} catch (PersistenceException e) {
log.error("Failed to create history view");
log.debug("failed to create history view: {}", e.getMessage());
session2.close();
throw new TableMalformedException("Failed to create history view");
}
session2.close();
execute(connection, tableMapper.tableToCreateHistoryViewRawQuery(entity));
/* save */
final Table table = tableRepository.save(entity);
log.info("Created table with id {}", table.getId());
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment