diff --git a/.invenio/feature_extract.ipynb b/.invenio/feature_extract.ipynb index a41f2154e379b3e5d4a15258e5b11daa8b8f27f4..dfa34c18584a7f8ae0dcb1a0dd65c3df13f780e5 100644 --- a/.invenio/feature_extract.ipynb +++ b/.invenio/feature_extract.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 51, "metadata": { "pycharm": { "name": "#%%\n" @@ -54,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 52, "metadata": { "pycharm": { "name": "#%%\n" @@ -80,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 53, "metadata": { "pycharm": { "name": "#%%\n" @@ -94,11 +94,11 @@ "{'authorities': [{'authority': 'ROLE_RESEARCHER'}],\n", " 'containers': None,\n", " 'databases': None,\n", - " 'email': 'martinweiseat@gmail.com',\n", + " 'email': 'martin.weise@tuwien.ac.at',\n", " 'email_verified': False,\n", " 'firstname': 'Martin',\n", " 'has_invenio_token': False,\n", - " 'id': 3,\n", + " 'id': 2,\n", " 'identifiers': None,\n", " 'lastname': 'Weise',\n", " 'titles_after': None,\n", @@ -117,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 54, "metadata": { "pycharm": { "name": "#%%\n" @@ -128,25 +128,25 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'created': datetime.datetime(2022, 7, 15, 7, 57, 17, 276000, tzinfo=tzutc()),\n", + "{'created': datetime.datetime(2022, 7, 16, 23, 7, 27, 694000, tzinfo=tzutc()),\n", " 'creator': {'authorities': None,\n", " 'containers': None,\n", " 'databases': None,\n", - " 'email': 'martinweiseat@gmail.com',\n", + " 'email': 'martin.weise@tuwien.ac.at',\n", " 'email_verified': False,\n", " 'firstname': 'Martin',\n", " 'has_invenio_token': None,\n", - " 'id': 3,\n", + " 'id': 2,\n", " 'identifiers': None,\n", " 'lastname': 'Weise',\n", " 'titles_after': None,\n", " 'titles_before': None,\n", " 'username': 'user'},\n", - " 'hash': '1c9babfb9fd1db38cf72e324a0a4bf4c6c35124d57693bd968c8f76bec9b9a1a',\n", - " 'id': 2,\n", - " 'internal_name': 'fda-userdb-ethmusmir-bee77de4-0413-11ed-9856-f3afdf8405f0',\n", + " 'hash': 'e82d1959b1077dc04803c424b7860c7294bb21436df89d1bbf967ed3ce28d4e8',\n", + " 'id': 5,\n", + " 'internal_name': 'fda-userdb-ethmusmir-0fa84f08-055c-11ed-926d-6130ea600442',\n", " 'is_public': None,\n", - " 'name': 'ethmusmir bee77de4-0413-11ed-9856-f3afdf8405f0'}\n" + " 'name': 'ethmusmir 0fa84f08-055c-11ed-926d-6130ea600442'}\n" ] } ], @@ -162,7 +162,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 55, "metadata": { "pycharm": { "name": "#%%\n" @@ -173,25 +173,25 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'created': datetime.datetime(2022, 7, 15, 7, 57, 17, 276000, tzinfo=tzutc()),\n", + "{'created': datetime.datetime(2022, 7, 16, 23, 7, 27, 694000, tzinfo=tzutc()),\n", " 'creator': {'authorities': None,\n", " 'containers': None,\n", " 'databases': None,\n", - " 'email': 'martinweiseat@gmail.com',\n", + " 'email': 'martin.weise@tuwien.ac.at',\n", " 'email_verified': False,\n", " 'firstname': 'Martin',\n", " 'has_invenio_token': None,\n", - " 'id': 3,\n", + " 'id': 2,\n", " 'identifiers': None,\n", " 'lastname': 'Weise',\n", " 'titles_after': None,\n", " 'titles_before': None,\n", " 'username': 'user'},\n", - " 'hash': '1c9babfb9fd1db38cf72e324a0a4bf4c6c35124d57693bd968c8f76bec9b9a1a',\n", - " 'id': 2,\n", - " 'internal_name': 'fda-userdb-ethmusmir-bee77de4-0413-11ed-9856-f3afdf8405f0',\n", + " 'hash': 'e82d1959b1077dc04803c424b7860c7294bb21436df89d1bbf967ed3ce28d4e8',\n", + " 'id': 5,\n", + " 'internal_name': 'fda-userdb-ethmusmir-0fa84f08-055c-11ed-926d-6130ea600442',\n", " 'is_public': None,\n", - " 'name': 'ethmusmir bee77de4-0413-11ed-9856-f3afdf8405f0'}\n" + " 'name': 'ethmusmir 0fa84f08-055c-11ed-926d-6130ea600442'}\n" ] } ], @@ -205,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 56, "metadata": { "pycharm": { "name": "#%%\n" @@ -217,42 +217,42 @@ "output_type": "stream", "text": [ "{'contact': None,\n", - " 'container': {'created': datetime.datetime(2022, 7, 15, 7, 57, 17, 276000, tzinfo=tzutc()),\n", + " 'container': {'created': datetime.datetime(2022, 7, 16, 23, 7, 27, 694000, tzinfo=tzutc()),\n", " 'databases': None,\n", - " 'hash': '1c9babfb9fd1db38cf72e324a0a4bf4c6c35124d57693bd968c8f76bec9b9a1a',\n", - " 'id': 2,\n", + " 'hash': 'e82d1959b1077dc04803c424b7860c7294bb21436df89d1bbf967ed3ce28d4e8',\n", + " 'id': 5,\n", " 'image': {'compiled': None,\n", - " 'date_formats': [{'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 903000, tzinfo=tzutc()),\n", + " 'date_formats': [{'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 212000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d',\n", " 'example': '2022-01-30',\n", " 'has_time': False,\n", " 'id': 1,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 909000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 221000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%Y',\n", " 'example': '30.01.2022',\n", " 'has_time': False,\n", " 'id': 2,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 912000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 225000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%y',\n", " 'example': '30.01.22',\n", " 'has_time': False,\n", " 'id': 3,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 914000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 229000, tzinfo=tzutc()),\n", " 'database_format': '%c/%d/%Y',\n", " 'example': '01/30/2022',\n", " 'has_time': False,\n", " 'id': 4,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 916000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 234000, tzinfo=tzutc()),\n", " 'database_format': '%c/%d/%y',\n", " 'example': '01/30/22',\n", " 'has_time': False,\n", " 'id': 5,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 919000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 237000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d '\n", " '%H:%i:%S.%f',\n", " 'example': '2022-01-30 13:44:25.0',\n", @@ -260,7 +260,7 @@ " 'id': 6,\n", " 'unix_format': 'yyyy-MM-dd '\n", " 'HH:mm:ss.SSSSSS'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 921000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 240000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d '\n", " '%H:%i:%S',\n", " 'example': '2022-01-30 13:44:25',\n", @@ -268,7 +268,7 @@ " 'id': 7,\n", " 'unix_format': 'yyyy-MM-dd '\n", " 'HH:mm:ss'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 923000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 244000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%Y '\n", " '%H:%i:%S',\n", " 'example': '30.01.2022 13:44:25',\n", @@ -302,21 +302,21 @@ " 'repository': 'mariadb',\n", " 'size': None,\n", " 'tag': '10.5'},\n", - " 'internal_name': 'fda-userdb-ethmusmir-bee77de4-0413-11ed-9856-f3afdf8405f0',\n", + " 'internal_name': 'fda-userdb-ethmusmir-0fa84f08-055c-11ed-926d-6130ea600442',\n", " 'ip_address': None,\n", " 'is_public': None,\n", - " 'name': 'ethmusmir bee77de4-0413-11ed-9856-f3afdf8405f0',\n", - " 'port': 35762,\n", + " 'name': 'ethmusmir 0fa84f08-055c-11ed-926d-6130ea600442',\n", + " 'port': 14458,\n", " 'state': None},\n", - " 'created': datetime.datetime(2022, 7, 15, 7, 57, 23, 75000, tzinfo=tzutc()),\n", + " 'created': datetime.datetime(2022, 7, 16, 23, 7, 33, 468000, tzinfo=tzutc()),\n", " 'creator': {'authorities': None,\n", " 'containers': None,\n", " 'databases': None,\n", - " 'email': 'martinweiseat@gmail.com',\n", + " 'email': 'martin.weise@tuwien.ac.at',\n", " 'email_verified': False,\n", " 'firstname': 'Martin',\n", " 'has_invenio_token': None,\n", - " 'id': 3,\n", + " 'id': 2,\n", " 'identifiers': None,\n", " 'lastname': 'Weise',\n", " 'titles_after': None,\n", @@ -324,52 +324,52 @@ " 'username': 'user'},\n", " 'deleted': None,\n", " 'description': 'ethmusmir',\n", - " 'exchange': 'ethmusmir_c23fc99c-0413-11ed-9856-f3afdf8405f0',\n", - " 'id': 2,\n", + " 'exchange': 'ethmusmir_12fb6852-055c-11ed-926d-6130ea600442',\n", + " 'id': 5,\n", " 'image': {'compiled': None,\n", - " 'date_formats': [{'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 903000, tzinfo=tzutc()),\n", + " 'date_formats': [{'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 212000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d',\n", " 'example': '2022-01-30',\n", " 'has_time': False,\n", " 'id': 1,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 909000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 221000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%Y',\n", " 'example': '30.01.2022',\n", " 'has_time': False,\n", " 'id': 2,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 912000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 225000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%y',\n", " 'example': '30.01.22',\n", " 'has_time': False,\n", " 'id': 3,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 914000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 229000, tzinfo=tzutc()),\n", " 'database_format': '%c/%d/%Y',\n", " 'example': '01/30/2022',\n", " 'has_time': False,\n", " 'id': 4,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 916000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 234000, tzinfo=tzutc()),\n", " 'database_format': '%c/%d/%y',\n", " 'example': '01/30/22',\n", " 'has_time': False,\n", " 'id': 5,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 919000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 237000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d %H:%i:%S.%f',\n", " 'example': '2022-01-30 13:44:25.0',\n", " 'has_time': True,\n", " 'id': 6,\n", " 'unix_format': 'yyyy-MM-dd HH:mm:ss.SSSSSS'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 921000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 240000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d %H:%i:%S',\n", " 'example': '2022-01-30 13:44:25',\n", " 'has_time': True,\n", " 'id': 7,\n", " 'unix_format': 'yyyy-MM-dd HH:mm:ss'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 923000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 244000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%Y %H:%i:%S',\n", " 'example': '30.01.2022 13:44:25',\n", " 'has_time': True,\n", @@ -401,11 +401,11 @@ " 'repository': 'mariadb',\n", " 'size': None,\n", " 'tag': '10.5'},\n", - " 'internal_name': 'ethmusmir_c23fc99c-0413-11ed-9856-f3afdf8405f0',\n", - " 'is_public': True,\n", + " 'internal_name': 'ethmusmir_12fb6852-055c-11ed-926d-6130ea600442',\n", + " 'is_public': False,\n", " 'language': None,\n", " 'license': None,\n", - " 'name': 'ethmusmir c23fc99c-0413-11ed-9856-f3afdf8405f0',\n", + " 'name': 'ethmusmir 12fb6852-055c-11ed-926d-6130ea600442',\n", " 'publication': None,\n", " 'publisher': None,\n", " 'subjects': None,\n", @@ -417,7 +417,7 @@ "response = database.create({\n", " \"name\": \"ethmusmir \" + str(uuid.uuid1()),\n", " \"description\": \"ethmusmir\",\n", - " \"is_public\": True\n", + " \"is_public\": False\n", "}, container_id)\n", "database_id = response.id\n", "print(response)" @@ -425,49 +425,49 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 57, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'contact': None,\n", - " 'container': {'created': datetime.datetime(2022, 7, 15, 7, 57, 17, 276000, tzinfo=tzutc()),\n", + " 'container': {'created': datetime.datetime(2022, 7, 16, 23, 7, 27, 694000, tzinfo=tzutc()),\n", " 'databases': None,\n", - " 'hash': '1c9babfb9fd1db38cf72e324a0a4bf4c6c35124d57693bd968c8f76bec9b9a1a',\n", - " 'id': 2,\n", + " 'hash': 'e82d1959b1077dc04803c424b7860c7294bb21436df89d1bbf967ed3ce28d4e8',\n", + " 'id': 5,\n", " 'image': {'compiled': None,\n", - " 'date_formats': [{'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 903000, tzinfo=tzutc()),\n", + " 'date_formats': [{'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 212000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d',\n", " 'example': '2022-01-30',\n", " 'has_time': False,\n", " 'id': 1,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 909000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 221000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%Y',\n", " 'example': '30.01.2022',\n", " 'has_time': False,\n", " 'id': 2,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 912000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 225000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%y',\n", " 'example': '30.01.22',\n", " 'has_time': False,\n", " 'id': 3,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 914000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 229000, tzinfo=tzutc()),\n", " 'database_format': '%c/%d/%Y',\n", " 'example': '01/30/2022',\n", " 'has_time': False,\n", " 'id': 4,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 916000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 234000, tzinfo=tzutc()),\n", " 'database_format': '%c/%d/%y',\n", " 'example': '01/30/22',\n", " 'has_time': False,\n", " 'id': 5,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 919000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 237000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d '\n", " '%H:%i:%S.%f',\n", " 'example': '2022-01-30 13:44:25.0',\n", @@ -475,7 +475,7 @@ " 'id': 6,\n", " 'unix_format': 'yyyy-MM-dd '\n", " 'HH:mm:ss.SSSSSS'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 921000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 240000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d '\n", " '%H:%i:%S',\n", " 'example': '2022-01-30 13:44:25',\n", @@ -483,7 +483,7 @@ " 'id': 7,\n", " 'unix_format': 'yyyy-MM-dd '\n", " 'HH:mm:ss'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 923000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 244000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%Y '\n", " '%H:%i:%S',\n", " 'example': '30.01.2022 13:44:25',\n", @@ -517,21 +517,21 @@ " 'repository': 'mariadb',\n", " 'size': None,\n", " 'tag': '10.5'},\n", - " 'internal_name': 'fda-userdb-ethmusmir-bee77de4-0413-11ed-9856-f3afdf8405f0',\n", + " 'internal_name': 'fda-userdb-ethmusmir-0fa84f08-055c-11ed-926d-6130ea600442',\n", " 'ip_address': None,\n", " 'is_public': None,\n", - " 'name': 'ethmusmir bee77de4-0413-11ed-9856-f3afdf8405f0',\n", - " 'port': 35762,\n", + " 'name': 'ethmusmir 0fa84f08-055c-11ed-926d-6130ea600442',\n", + " 'port': 14458,\n", " 'state': None},\n", - " 'created': datetime.datetime(2022, 7, 15, 7, 57, 23, 75000, tzinfo=tzutc()),\n", + " 'created': datetime.datetime(2022, 7, 16, 23, 7, 33, 468000, tzinfo=tzutc()),\n", " 'creator': {'authorities': None,\n", " 'containers': None,\n", " 'databases': None,\n", - " 'email': 'martinweiseat@gmail.com',\n", + " 'email': 'martin.weise@tuwien.ac.at',\n", " 'email_verified': False,\n", " 'firstname': 'Martin',\n", " 'has_invenio_token': None,\n", - " 'id': 3,\n", + " 'id': 2,\n", " 'identifiers': None,\n", " 'lastname': 'Weise',\n", " 'titles_after': None,\n", @@ -542,52 +542,52 @@ " 'https://github.com/ketchupok/ethmusmir applied on a remixed '\n", " 'recording of the SeFiRe field recordings dataset '\n", " 'https://github.com/matijama/field-recording-db',\n", - " 'exchange': 'ethmusmir_c23fc99c-0413-11ed-9856-f3afdf8405f0',\n", - " 'id': 2,\n", + " 'exchange': 'ethmusmir_12fb6852-055c-11ed-926d-6130ea600442',\n", + " 'id': 5,\n", " 'image': {'compiled': None,\n", - " 'date_formats': [{'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 903000, tzinfo=tzutc()),\n", + " 'date_formats': [{'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 212000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d',\n", " 'example': '2022-01-30',\n", " 'has_time': False,\n", " 'id': 1,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 909000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 221000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%Y',\n", " 'example': '30.01.2022',\n", " 'has_time': False,\n", " 'id': 2,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 912000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 225000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%y',\n", " 'example': '30.01.22',\n", " 'has_time': False,\n", " 'id': 3,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 914000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 229000, tzinfo=tzutc()),\n", " 'database_format': '%c/%d/%Y',\n", " 'example': '01/30/2022',\n", " 'has_time': False,\n", " 'id': 4,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 916000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 234000, tzinfo=tzutc()),\n", " 'database_format': '%c/%d/%y',\n", " 'example': '01/30/22',\n", " 'has_time': False,\n", " 'id': 5,\n", " 'unix_format': 'yyyy-MM-dd'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 919000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 237000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d %H:%i:%S.%f',\n", " 'example': '2022-01-30 13:44:25.0',\n", " 'has_time': True,\n", " 'id': 6,\n", " 'unix_format': 'yyyy-MM-dd HH:mm:ss.SSSSSS'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 921000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 240000, tzinfo=tzutc()),\n", " 'database_format': '%Y-%c-%d %H:%i:%S',\n", " 'example': '2022-01-30 13:44:25',\n", " 'has_time': True,\n", " 'id': 7,\n", " 'unix_format': 'yyyy-MM-dd HH:mm:ss'},\n", - " {'created_at': datetime.datetime(2022, 7, 15, 6, 11, 37, 923000, tzinfo=tzutc()),\n", + " {'created_at': datetime.datetime(2022, 7, 16, 22, 53, 7, 244000, tzinfo=tzutc()),\n", " 'database_format': '%d.%c.%Y %H:%i:%S',\n", " 'example': '30.01.2022 13:44:25',\n", " 'has_time': True,\n", @@ -619,12 +619,12 @@ " 'repository': 'mariadb',\n", " 'size': None,\n", " 'tag': '10.5'},\n", - " 'internal_name': 'ethmusmir_c23fc99c-0413-11ed-9856-f3afdf8405f0',\n", - " 'is_public': True,\n", + " 'internal_name': 'ethmusmir_12fb6852-055c-11ed-926d-6130ea600442',\n", + " 'is_public': False,\n", " 'language': 'en',\n", " 'license': {'identifier': 'Apache-2.0',\n", " 'uri': 'https://opensource.org/licenses/Apache-2.0'},\n", - " 'name': 'ethmusmir c23fc99c-0413-11ed-9856-f3afdf8405f0',\n", + " 'name': 'ethmusmir 12fb6852-055c-11ed-926d-6130ea600442',\n", " 'publication': '2022-07-17',\n", " 'publisher': 'Technical University of Vienna',\n", " 'subjects': [],\n", @@ -641,7 +641,7 @@ " \"uri\": \"https://opensource.org/licenses/Apache-2.0\"\n", " },\n", " \"language\": \"en\",\n", - " \"is_public\": True,\n", + " \"is_public\": False,\n", " \"publication\": \"2022-07-17\"\n", "}, container_id, database_id)\n", "print(response)" @@ -655,7 +655,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 58, "metadata": { "pycharm": { "name": "#%%\n" @@ -666,7 +666,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'id': 2, 'internal_name': 'feature_extraction', 'name': 'Feature Extraction'}\n" + "{'id': 5, 'internal_name': 'feature_extraction', 'name': 'Feature Extraction'}\n" ] } ], @@ -700,7 +700,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 59, "metadata": { "pycharm": { "name": "#%%\n" @@ -717,7 +717,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 60, "metadata": { "pycharm": { "name": "#%%\n" @@ -729,7 +729,7 @@ "output_type": "stream", "text": [ "... save file contents from https://test.researchdata.tuwien.ac.at/api/records/xksnz-cfg98/files/audio.wav/content\n", - "... extracted {'track': {'parts': [{'start': 0, 'content': 'instrumental', 'duration': 5}, {'start': 5, 'content': 'speech', 'duration': 4}, {'start': 9, 'content': 'choir singing', 'duration': 6}, {'start': 15, 'content': 'solo singing', 'duration': 4}], 'tempo': 'nan', 'duration': 20.323265306122448}}\n", + "... extracted {'track': {'parts': [{'content': 'instrumental', 'duration': 5, 'start': 0}, {'content': 'speech', 'duration': 4, 'start': 5}, {'content': 'choir singing', 'duration': 6, 'start': 9}, {'content': 'solo singing', 'duration': 4, 'start': 15}], 'duration': 20.323265306122448, 'tempo': 'nan'}}\n", "1\n", "1\n", "1\n", @@ -754,7 +754,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 61, "metadata": { "pycharm": { "name": "#%%\n" @@ -782,7 +782,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 62, "metadata": { "pycharm": { "name": "#%%\n" @@ -793,39 +793,39 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'cid': 2,\n", - " 'created': datetime.datetime(2022, 7, 15, 7, 57, 31, 152000, tzinfo=tzutc()),\n", + "{'cid': 5,\n", + " 'created': datetime.datetime(2022, 7, 16, 23, 7, 43, 70000, tzinfo=tzutc()),\n", " 'creator': {'authorities': None,\n", " 'containers': None,\n", " 'databases': None,\n", - " 'email': 'martinweiseat@gmail.com',\n", + " 'email': 'martin.weise@tuwien.ac.at',\n", " 'email_verified': False,\n", " 'firstname': 'Martin',\n", " 'has_invenio_token': None,\n", - " 'id': 3,\n", + " 'id': 2,\n", " 'identifiers': None,\n", " 'lastname': 'Weise',\n", " 'titles_after': None,\n", " 'titles_before': None,\n", " 'username': 'user'},\n", " 'creators': [{'affiliation': 'TU Wien',\n", - " 'created': datetime.datetime(2022, 7, 15, 7, 57, 31, 168000, tzinfo=tzutc()),\n", - " 'id': 2,\n", - " 'last_modified': datetime.datetime(2022, 7, 15, 7, 57, 31, 168000, tzinfo=tzutc()),\n", + " 'created': datetime.datetime(2022, 7, 16, 23, 7, 43, 82000, tzinfo=tzutc()),\n", + " 'id': 5,\n", + " 'last_modified': datetime.datetime(2022, 7, 16, 23, 7, 43, 82000, tzinfo=tzutc()),\n", " 'name': 'Weise, Martin',\n", " 'orcid': '0000-0003-4216-302X'},\n", " {'affiliation': 'TU Wien',\n", - " 'created': datetime.datetime(2022, 7, 15, 7, 57, 31, 176000, tzinfo=tzutc()),\n", - " 'id': 3,\n", - " 'last_modified': datetime.datetime(2022, 7, 15, 7, 57, 31, 176000, tzinfo=tzutc()),\n", + " 'created': datetime.datetime(2022, 7, 16, 23, 7, 43, 87000, tzinfo=tzutc()),\n", + " 'id': 6,\n", + " 'last_modified': datetime.datetime(2022, 7, 16, 23, 7, 43, 87000, tzinfo=tzutc()),\n", " 'name': 'Rauber, Andreas',\n", " 'orcid': '0000-0002-9272-6225'}],\n", - " 'dbid': 2,\n", + " 'dbid': 5,\n", " 'description': 'description',\n", " 'doi': None,\n", - " 'execution': datetime.datetime(2022, 7, 15, 5, 57, 30, tzinfo=tzutc()),\n", - " 'id': 2,\n", - " 'last_modified': datetime.datetime(2022, 7, 15, 7, 57, 31, 152000, tzinfo=tzutc()),\n", + " 'execution': datetime.datetime(2022, 7, 16, 23, 7, 42, tzinfo=tzutc()),\n", + " 'id': 3,\n", + " 'last_modified': datetime.datetime(2022, 7, 16, 23, 7, 43, 70000, tzinfo=tzutc()),\n", " 'publication': '2022-07-16',\n", " 'qid': 1,\n", " 'query': 'SELECT `content`, `start`, `duration` FROM `feature_extraction`',\n", @@ -872,13 +872,13 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 63, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "http://localhost:3000/pid/2\n" + "http://localhost:3000/pid/3\n" ] } ], diff --git a/.rabbitmq/api_broker/BrokerServiceClient.py b/.rabbitmq/api_broker/BrokerServiceClient.py new file mode 100644 index 0000000000000000000000000000000000000000..484837109103a5d287cd3e8c7cd41db95e8274e2 --- /dev/null +++ b/.rabbitmq/api_broker/BrokerServiceClient.py @@ -0,0 +1,28 @@ +#!/bin/env python3 +import pika +import json + + +class BrokerServiceClient: + + def __init__(self, exchange=None, routing_key=None, host=None, username=None, password=None): + self.exchange = exchange + self.routing_key = routing_key + self.host = host + self.username = username + self.password = password + + def send(self, data): + creds = pika.credentials.PlainCredentials(self.username, self.password) + connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=creds)) + channel = connection.channel() + dump = json.dumps(data) + channel.basic_publish(exchange=self.exchange, + routing_key=self.routing_key, + body=bytes(json.dumps(data), encoding='utf8')) + method, properties, body = channel.basic_get(queue=self.routing_key) + connection.close() + if method is not None: + return False + else: + return True diff --git a/.rabbitmq/api_broker/__pycache__/BrokerServiceClient.cpython-310.pyc b/.rabbitmq/api_broker/__pycache__/BrokerServiceClient.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e211fe3cda3d7b5e838dfb174395110fe8d9b4e5 Binary files /dev/null and b/.rabbitmq/api_broker/__pycache__/BrokerServiceClient.cpython-310.pyc differ diff --git a/.rabbitmq/publish.ipynb b/.rabbitmq/publish.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f46656bc2d456758e15dea1546666ec0e9516b1a --- /dev/null +++ b/.rabbitmq/publish.ipynb @@ -0,0 +1,90 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 203, + "metadata": { + "collapsed": true, + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "from api_broker.BrokerServiceClient import BrokerServiceClient\n", + "\n", + "exchange = \"airquality\"\n", + "routing_key = \"airquality\"\n", + "host = \"localhost\"\n", + "username = \"user\"\n", + "password = \"user\"\n", + "\n", + "broker = BrokerServiceClient(exchange=exchange, routing_key=routing_key, host=host, username=username, password=password)" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Ingest AMQP" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%% md\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 204, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], + "source": [ + "response = broker.send({\n", + " \"date\": \"2021-07-17\",\n", + " \"location\": \"Melaka\",\n", + " \"parameter\": \"CO\",\n", + " \"interval\": \"h1\",\n", + " \"unit\": \"mg/m3\",\n", + " \"value\": 0.3,\n", + " \"status\": \"tentative\"\n", + "})\n", + "print(response)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/.rabbitmq/requirements.txt b/.rabbitmq/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..700b460421a7eb5ea93de31049ff011f1d1ed984 --- /dev/null +++ b/.rabbitmq/requirements.txt @@ -0,0 +1 @@ +pika==1.3.0 \ No newline at end of file diff --git a/fda-analyse-service/determine_dt.py b/fda-analyse-service/determine_dt.py index 18e14619abfe2f3c7988a93d524ec7333a213ceb..9dc8c26f22c4a815369153d9186bdad3d43b50f7 100644 --- a/fda-analyse-service/determine_dt.py +++ b/fda-analyse-service/determine_dt.py @@ -13,12 +13,13 @@ import json import csv import messytables, pandas as pd from messytables import CSVTableSet, type_guess, \ - headers_guess, headers_processor, offset_processor + headers_guess, headers_processor, offset_processor -def determine_datatypes(path, enum=False, enum_tol=0.0001,separator = None): -# Use option enum=True for searching Postgres ENUM Types in CSV file. Remark -# Enum is not SQL standard, hence, it might not be supported by all db-engines. -# However, it can be used in Postgres and MySQL. + +def determine_datatypes(path, enum=False, enum_tol=0.0001, separator=None): + # Use option enum=True for searching Postgres ENUM Types in CSV file. Remark + # Enum is not SQL standard, hence, it might not be supported by all db-engines. + # However, it can be used in Postgres and MySQL. fh = open(path, 'rb') if separator == None: with open(path) as csvfile: @@ -26,7 +27,7 @@ def determine_datatypes(path, enum=False, enum_tol=0.0001,separator = None): separator = dialect.delimiter # Load a file object: - table_set = CSVTableSet(fh,delimiter=separator) + table_set = CSVTableSet(fh, delimiter=separator) # A table set is a collection of tables: row_set = table_set.tables[0] @@ -44,46 +45,49 @@ def determine_datatypes(path, enum=False, enum_tol=0.0001,separator = None): r = {} # list of rows - if enum ==True: - rows = pd.read_csv(path,sep=separator,header=offset) + if enum == True: + rows = pd.read_csv(path, sep=separator, header=offset) n = len(rows) - for i in range(0,(len(types))): + for i in range(0, (len(types))): if type(types[i]) == messytables.types.BoolType: r[headers[i]] = "Boolean" elif type(types[i]) == messytables.types.IntegerType: r[headers[i]] = "Integer" + elif type(types[i]) == messytables.types.FloatType: + r[headers[i]] = "Decimal" elif type(types[i]) == messytables.types.DateType: if ("S" in str(types[i])): r[headers[i]] = "Timestamp" else: r[headers[i]] = "Date" elif type(types[i]) == messytables.types.DecimalType: - r[headers[i]] = "Numeric" - else : + r[headers[i]] = "Decimal" + else: if enum == True: enum_set = set() - m=0 + m = 0 is_enum = True - for elem in range(0,n): - if (m < enum_tol*n): - enum_set.add(rows.iloc[elem,i]) - else: + for elem in range(0, n): + if (m < enum_tol * n): + enum_set.add(rows.iloc[elem, i]) + else: is_enum = False break m = len(enum_set) if is_enum: enum_set.discard(None) r[headers[i]] = {"Enum": list(enum_set)} - else: + else: r[headers[i]] = "Text" - else: + else: r[headers[i]] = "Text" - s ={ 'columns' : r , 'separator': separator} + s = {'columns': r, 'separator': separator} return json.dumps(s) + """ Example output: { diff --git a/fda-query-service/pom.xml b/fda-query-service/pom.xml index 56079587262f6263324f10306bfceed5c186dedb..64308c48222d924c175be0d799340fe8f0eccccb 100644 --- a/fda-query-service/pom.xml +++ b/fda-query-service/pom.xml @@ -98,6 +98,12 @@ <groupId>org.springframework.data</groupId> <artifactId>spring-data-elasticsearch</artifactId> </dependency> + <!-- AMPQ --> + <dependency> + <groupId>com.rabbitmq</groupId> + <artifactId>amqp-client</artifactId> + <version>${rabbit-amqp-client.version}</version> + </dependency> <!-- Docker --> <dependency> <groupId>com.github.docker-java</groupId> diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/FdaQueryServiceApplication.java b/fda-query-service/rest-service/src/main/java/at/tuwien/FdaQueryServiceApplication.java index 46aa6e41262c65467fdeee77045c1b059fafda8b..d191795cefa4309a863106faf4909a3ef1382c59 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/FdaQueryServiceApplication.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/FdaQueryServiceApplication.java @@ -6,11 +6,13 @@ import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; import org.springframework.data.jpa.repository.config.EnableJpaAuditing; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.transaction.annotation.EnableTransactionManagement; @EnableJpaAuditing @SpringBootApplication @EnableTransactionManagement +@EnableScheduling @EntityScan(basePackages = "at.tuwien.entities") @EnableElasticsearchRepositories(basePackages = {"at.tuwien.repository.elastic"}) @EnableJpaRepositories(basePackages = {"at.tuwien.repository.jpa"}) diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/AbstractEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/AbstractEndpoint.java index c744e90ea56b1f8834df15657f6da2c35ddd3eed..bb7c46e45234bb9fd114bad4f75fe037d7730e25 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/AbstractEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/AbstractEndpoint.java @@ -33,11 +33,11 @@ public abstract class AbstractEndpoint { this.identifierService = identifierService; } - protected Boolean hasDatabasePermission(Long databaseId, Long tableId, String permissionCode, + protected Boolean hasDatabasePermission(Long containerId, Long databaseId, Long tableId, String permissionCode, Principal principal) { final Table table; try { - table = tableService.find(databaseId, tableId); + table = tableService.find(containerId, databaseId, tableId); } catch (DatabaseNotFoundException e) { log.debug("failed to find database with id {}", databaseId); return false; diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ExportEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ExportEndpoint.java index 64b694f64ce1671157a8038457d9e97361bab43b..2db195215171969ddeaf6597bb4f7a67f585b30b 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ExportEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/ExportEndpoint.java @@ -46,7 +46,7 @@ public class ExportEndpoint extends AbstractEndpoint { throws TableNotFoundException, DatabaseConnectionException, TableMalformedException, DatabaseNotFoundException, ImageNotSupportedException, PaginationException, ContainerNotFoundException, FileStorageException, NotAllowedException { - if (!hasDatabasePermission(databaseId, tableId, "DATA_EXPORT", principal)) { + if (!hasDatabasePermission(id, databaseId, tableId, "DATA_EXPORT", principal)) { log.error("Missing data export permission"); throw new NotAllowedException("Missing data export permission"); } diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java index 8614fff0284437350226c8b56dac1962c0f578c7..f6a57a34364bd13f88244cee16e7eb56b68383c6 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableDataEndpoint.java @@ -50,9 +50,9 @@ public class TableDataEndpoint extends AbstractEndpoint { @NotNull @PathVariable("tableId") Long tableId, @NotNull @Valid @RequestBody TableCsvDto data, @NotNull Principal principal) - throws TableNotFoundException, DatabaseNotFoundException, FileStorageException, TableMalformedException, + throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException, ImageNotSupportedException, ContainerNotFoundException, NotAllowedException { - if (!hasDatabasePermission(databaseId, tableId, "DATA_INSERT", principal)) { + if (!hasDatabasePermission(containerId, databaseId, tableId, "DATA_INSERT", principal)) { log.error("Missing data insert permission"); throw new NotAllowedException("Missing data insert permission"); } @@ -70,7 +70,7 @@ public class TableDataEndpoint extends AbstractEndpoint { @NotNull Principal principal) throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException, ImageNotSupportedException, NotAllowedException { - if (!hasDatabasePermission(databaseId, tableId, "DATA_UPDATE", principal)) { + if (!hasDatabasePermission(containerId, databaseId, tableId, "DATA_UPDATE", principal)) { log.error("Missing data update permission"); throw new NotAllowedException("Missing data update permission"); } @@ -88,7 +88,7 @@ public class TableDataEndpoint extends AbstractEndpoint { @NotNull Principal principal) throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException, ImageNotSupportedException, TupleDeleteException, NotAllowedException { - if (!hasDatabasePermission(databaseId, tableId, "DATA_DELETE", principal)) { + if (!hasDatabasePermission(containerId, databaseId, tableId, "DATA_DELETE", principal)) { log.error("Missing data delete permission"); throw new NotAllowedException("Missing data delete permission"); } @@ -107,7 +107,7 @@ public class TableDataEndpoint extends AbstractEndpoint { @NotNull Principal principal) throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException, ImageNotSupportedException, ContainerNotFoundException, NotAllowedException { - if (!hasDatabasePermission(databaseId, tableId, "DATA_INSERT", principal)) { + if (!hasDatabasePermission(containerId, databaseId, tableId, "DATA_INSERT", principal)) { log.error("Missing data insert permission"); throw new NotAllowedException("Missing data insert permission"); } @@ -129,7 +129,7 @@ public class TableDataEndpoint extends AbstractEndpoint { throws TableNotFoundException, DatabaseNotFoundException, DatabaseConnectionException, ImageNotSupportedException, TableMalformedException, PaginationException, ContainerNotFoundException, QueryStoreException, NotAllowedException { - if (!hasDatabasePermission(databaseId, tableId, "DATA_VIEW", principal)) { + if (!hasDatabasePermission(containerId, databaseId, tableId, "DATA_VIEW", principal)) { log.error("Missing data view permission"); throw new NotAllowedException("Missing data view permission"); } diff --git a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableHistoryEndpoint.java b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableHistoryEndpoint.java index fb2932a67a683c65ba6f8feabe71929d8ecc8b4e..dd65407cee41de3682c4e08629ba6f29545dd9d5 100644 --- a/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableHistoryEndpoint.java +++ b/fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/TableHistoryEndpoint.java @@ -35,7 +35,7 @@ public class TableHistoryEndpoint { @NotNull @PathVariable("databaseId") Long databaseId, @NotNull @PathVariable("tableId") Long tableId) throws TableNotFoundException, QueryMalformedException, DatabaseNotFoundException { - final List<TableHistoryDto> history = tableService.findHistory(databaseId, tableId); + final List<TableHistoryDto> history = tableService.findHistory(containerId, databaseId, tableId); return ResponseEntity.ok(history); } diff --git a/fda-query-service/rest-service/src/main/resources/application-docker.yml b/fda-query-service/rest-service/src/main/resources/application-docker.yml index c8f4ec698f41d68b4b6ebfc062e4a9585c3b1d17..1f8171d23806dde9b5ff6f1e649bb3f8fe358f1a 100644 --- a/fda-query-service/rest-service/src/main/resources/application-docker.yml +++ b/fda-query-service/rest-service/src/main/resources/application-docker.yml @@ -18,6 +18,10 @@ spring: time_zone: Europe/Vienna application: name: fda-query-service + rabbitmq: + host: fda-broker-service + username: fda + password: fda cloud: loadbalancer.ribbon.enabled: false server.port: 9093 diff --git a/fda-query-service/rest-service/src/main/resources/application.yml b/fda-query-service/rest-service/src/main/resources/application.yml index 067cdd9dbc6933ada214ad62ebdc8dd9761374ad..b8d3af3a2750bb1aaa09a88dafd409bbdaca89f3 100644 --- a/fda-query-service/rest-service/src/main/resources/application.yml +++ b/fda-query-service/rest-service/src/main/resources/application.yml @@ -18,6 +18,10 @@ spring: time_zone: Europe/Vienna application: name: fda-query-service + rabbitmq: + host: localhost + username: fda + password: fda cloud: loadbalancer.ribbon.enabled: false server.port: 9093 diff --git a/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceUnitTest.java b/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceUnitTest.java index bd8dce979f01b0499077d9570736318ee7d778a2..f6b835fc30011ba3ba340d9476e924d18ad0835d 100644 --- a/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceUnitTest.java +++ b/fda-query-service/rest-service/src/test/java/at/tuwien/service/QueryServiceUnitTest.java @@ -134,7 +134,7 @@ public class QueryServiceUnitTest extends BaseUnitTest { /* mock */ when(databaseRepository.findById(DATABASE_1_ID)) .thenReturn(Optional.of(DATABASE_1)); - when(tableRepository.findOne(DATABASE_1_ID, TABLE_1_ID)) + when(tableRepository.find(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID)) .thenReturn(Optional.of(TABLE_1)); /* test */ @@ -149,7 +149,7 @@ public class QueryServiceUnitTest extends BaseUnitTest { /* mock */ when(databaseRepository.findById(DATABASE_1_ID)) .thenReturn(Optional.of(DATABASE_1)); - when(tableRepository.findOne(DATABASE_1_ID, TABLE_1_ID)) + when(tableRepository.find(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID)) .thenReturn(Optional.empty()); /* test */ @@ -166,7 +166,7 @@ public class QueryServiceUnitTest extends BaseUnitTest { /* mock */ when(databaseRepository.findById(DATABASE_1_ID)) .thenReturn(Optional.empty()); - when(tableRepository.findOne(DATABASE_1_ID, TABLE_1_ID)) + when(tableRepository.find(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID)) .thenReturn(Optional.of(TABLE_1)); /* test */ @@ -184,7 +184,7 @@ public class QueryServiceUnitTest extends BaseUnitTest { /* mock */ when(databaseRepository.findById(DATABASE_1_ID)) .thenReturn(Optional.of(DATABASE_1)); - when(tableRepository.findOne(DATABASE_1_ID, TABLE_1_ID)) + when(tableRepository.find(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID)) .thenReturn(Optional.of(TABLE_1)); /* test */ @@ -201,7 +201,7 @@ public class QueryServiceUnitTest extends BaseUnitTest { /* mock */ when(databaseRepository.findById(DATABASE_1_ID)) .thenReturn(Optional.of(DATABASE_1)); - when(tableRepository.findOne(DATABASE_1_ID, TABLE_1_ID)) + when(tableRepository.find(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID)) .thenReturn(Optional.of(TABLE_1)); /* test */ @@ -217,7 +217,7 @@ public class QueryServiceUnitTest extends BaseUnitTest { /* mock */ when(databaseRepository.findById(DATABASE_1_ID)) .thenReturn(Optional.of(DATABASE_1)); - when(tableRepository.findOne(DATABASE_1_ID, TABLE_1_ID)) + when(tableRepository.find(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID)) .thenReturn(Optional.of(TABLE_1)); /* test */ diff --git a/fda-query-service/rest-service/src/test/java/at/tuwien/service/TableServiceTest.java b/fda-query-service/rest-service/src/test/java/at/tuwien/service/TableServiceTest.java index d435bdcfc534b0b6436c58ade80beb40d1c0d81a..0a33960b59f4b6deecf961d1e0775eb2e96af920 100644 --- a/fda-query-service/rest-service/src/test/java/at/tuwien/service/TableServiceTest.java +++ b/fda-query-service/rest-service/src/test/java/at/tuwien/service/TableServiceTest.java @@ -90,7 +90,7 @@ public class TableServiceTest extends BaseUnitTest { public void findAll_succeeds() throws TableNotFoundException, DatabaseNotFoundException { /* mock */ - final List<TableColumn> response = tableService.find(DATABASE_1_ID, TABLE_1_ID) + final List<TableColumn> response = tableService.find(CONTAINER_1_ID, DATABASE_1_ID, TABLE_1_ID) .getColumns(); /* test */ diff --git a/fda-query-service/services/src/main/java/at/tuwien/config/AmqpConfig.java b/fda-query-service/services/src/main/java/at/tuwien/config/AmqpConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..93da331dc00afce2129ea2a51f52630e1f2747a9 --- /dev/null +++ b/fda-query-service/services/src/main/java/at/tuwien/config/AmqpConfig.java @@ -0,0 +1,42 @@ +package at.tuwien.config; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +@Slf4j +@Getter +@Configuration +public class AmqpConfig { + + @Value("${spring.rabbitmq.host}") + private String ampqHost; + + @Value("${spring.rabbitmq.username}") + private String amqpUsername; + + @Value("${spring.rabbitmq.password}") + private String amqpPassword; + + @Bean + public Channel getChannel() throws IOException, TimeoutException { + final ConnectionFactory factory = new ConnectionFactory(); + factory.setHost(ampqHost); + factory.setUsername(amqpUsername); + factory.setPassword(amqpPassword); + factory.setAutomaticRecoveryEnabled(true); + factory.setTopologyRecoveryEnabled(true); + factory.setNetworkRecoveryInterval(10000) /* attempt recovery every 10 seconds */; + final Connection connection = factory.newConnection(); + return connection.createChannel(); + } + +} diff --git a/fda-query-service/services/src/main/java/at/tuwien/config/ReadyConfig.java b/fda-query-service/services/src/main/java/at/tuwien/config/ReadyConfig.java index 0bee3b961edd4ca456f0243c8eede630a4a54716..2250fa50884df3f47b0b063975aea74f06203f80 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/config/ReadyConfig.java +++ b/fda-query-service/services/src/main/java/at/tuwien/config/ReadyConfig.java @@ -1,6 +1,7 @@ package at.tuwien.config; import com.google.common.io.Files; +import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.Configuration; @@ -9,6 +10,7 @@ import org.springframework.context.event.EventListener; import java.io.File; import java.io.IOException; +@Log4j2 @Configuration public class ReadyConfig { diff --git a/fda-query-service/services/src/main/java/at/tuwien/exception/AmqpException.java b/fda-query-service/services/src/main/java/at/tuwien/exception/AmqpException.java new file mode 100644 index 0000000000000000000000000000000000000000..6af0750d6f5089a8442a7159eac2076462df2825 --- /dev/null +++ b/fda-query-service/services/src/main/java/at/tuwien/exception/AmqpException.java @@ -0,0 +1,21 @@ +package at.tuwien.exception; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseStatus(code = HttpStatus.NOT_ACCEPTABLE) +public class AmqpException extends Exception { + + public AmqpException(String msg) { + super(msg); + } + + public AmqpException(String msg, Throwable thr) { + super(msg, thr); + } + + public AmqpException(Throwable thr) { + super(thr); + } + +} diff --git a/fda-query-service/services/src/main/java/at/tuwien/repository/jpa/TableRepository.java b/fda-query-service/services/src/main/java/at/tuwien/repository/jpa/TableRepository.java index e4b4301ef96f06b3813a4f8291e20237b2890f28..7a19c1f575891bd5fea52a4de87ea62b35c027bb 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/repository/jpa/TableRepository.java +++ b/fda-query-service/services/src/main/java/at/tuwien/repository/jpa/TableRepository.java @@ -11,7 +11,8 @@ import java.util.Optional; @Repository public interface TableRepository extends JpaRepository<Table, Long> { - @Query(value = "select t from Table t where t.database.id = :dbid and t.id = :tid") - Optional<Table> findOne(@Param("dbid") Long databaseId, @Param("tid") Long tableId); + @Query(value = "select t from Table t where t.database.container.id = :containerId and t.database.id = :databaseId and t.id = :tableId") + Optional<Table> find(@Param("containerId") Long containerId, @Param("databaseId") Long databaseId, + @Param("tableId") Long tableId); } diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/MessageQueueService.java b/fda-query-service/services/src/main/java/at/tuwien/service/MessageQueueService.java new file mode 100644 index 0000000000000000000000000000000000000000..89822c2d0885abb143e3962df4e66d8a561e4b55 --- /dev/null +++ b/fda-query-service/services/src/main/java/at/tuwien/service/MessageQueueService.java @@ -0,0 +1,6 @@ +package at.tuwien.service; + +public interface MessageQueueService { + + void renewConsumers(); +} diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/TableService.java b/fda-query-service/services/src/main/java/at/tuwien/service/TableService.java index 79f6378441f1778db3814b638f1d6635afc5d17c..08048f15aae34e58734d33fb96125278fb7d2853 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/service/TableService.java +++ b/fda-query-service/services/src/main/java/at/tuwien/service/TableService.java @@ -5,6 +5,7 @@ import at.tuwien.entities.database.table.Table; import at.tuwien.exception.DatabaseNotFoundException; import at.tuwien.exception.QueryMalformedException; import at.tuwien.exception.TableNotFoundException; +import org.springframework.transaction.annotation.Transactional; import java.util.List; @@ -13,24 +14,33 @@ public interface TableService { /** * Find a table in the metadata database by database-table id tuple * - * @param databaseId The database id. - * @param tableId The table id. + * @param containerId The container id. + * @param databaseId The database id. + * @param tableId The table id. * @return The database. * @throws DatabaseNotFoundException The database is not found. * @throws TableNotFoundException The table is not found. */ - Table find(Long databaseId, Long tableId) throws DatabaseNotFoundException, TableNotFoundException; + Table find(Long containerId, Long databaseId, Long tableId) throws DatabaseNotFoundException, TableNotFoundException; + + /** + * Finds all tables in the metdata database. + * + * @return The list of tables. + */ + List<Table> findAll(); /** * Find the table history. * - * @param databaseId The database id. - * @param tableId The table id. + * @param containerId The container id. + * @param databaseId The database id. + * @param tableId The table id. * @return The history as a list, if successful. * @throws QueryMalformedException The query is malformed. * @throws DatabaseNotFoundException The database is not found. * @throws TableNotFoundException The table is not found. */ - List<TableHistoryDto> findHistory(Long databaseId, Long tableId) + List<TableHistoryDto> findHistory(Long containerId, Long databaseId, Long tableId) throws DatabaseNotFoundException, QueryMalformedException, TableNotFoundException; } diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java b/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java index d2de04538f388aceb24e874735fea7f023192995..10ce827b9e4405a1021955a040ce08d915fba6df 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java +++ b/fda-query-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java @@ -127,7 +127,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService ContainerNotFoundException { /* find */ final Database database = databaseService.find(databaseId); - final Table table = tableService.find(databaseId, tableId); + final Table table = tableService.find(containerId, databaseId, tableId); /* run query */ final long startSession = System.currentTimeMillis(); final SessionFactory factory = getSessionFactory(database, true); @@ -168,7 +168,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService FileStorageException { /* find */ final Database database = databaseService.find(databaseId); - final Table table = tableService.find(databaseId, tableId); + final Table table = tableService.find(containerId, databaseId, tableId); /* run query */ final long startSession = System.currentTimeMillis(); final SessionFactory factory = getSessionFactory(database, true); @@ -251,7 +251,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService TableMalformedException, ImageNotSupportedException { /* find */ final Database database = databaseService.find(databaseId); - final Table table = tableService.find(databaseId, tableId); + final Table table = tableService.find(containerId, databaseId, tableId); /* run query */ final long startSession = System.currentTimeMillis(); final SessionFactory factory = getSessionFactory(database, false); @@ -285,7 +285,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService TableNotFoundException, ContainerNotFoundException { /* find */ final Database database = databaseService.find(databaseId); - final Table table = tableService.find(databaseId, tableId); + final Table table = tableService.find(containerId, databaseId, tableId); /* run query */ if (data.getData().size() == 0) return null; final long startSession = System.currentTimeMillis(); @@ -308,7 +308,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService TableNotFoundException { /* find */ final Database database = databaseService.find(databaseId); - final Table table = tableService.find(databaseId, tableId); + final Table table = tableService.find(containerId, databaseId, tableId); /* run query */ if (data.getData().size() == 0 || data.getKeys().size() == 0) return null; final long startSession = System.currentTimeMillis(); @@ -333,7 +333,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService TableNotFoundException, TupleDeleteException { /* find */ final Database database = databaseService.find(databaseId); - final Table table = tableService.find(databaseId, tableId); + final Table table = tableService.find(containerId, databaseId, tableId); /* run query */ if (data.getKeys().size() == 0) return; final long startSession = System.currentTimeMillis(); @@ -374,7 +374,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService TableNotFoundException, ContainerNotFoundException { /* find */ final Database database = databaseService.find(databaseId); - final Table table = tableService.find(databaseId, tableId); + final Table table = tableService.find(containerId, databaseId, tableId); /* run query */ final long startSession = System.currentTimeMillis(); log.debug("opened hibernate session in {} ms", System.currentTimeMillis() - startSession); diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/impl/RabbitMqServiceImpl.java b/fda-query-service/services/src/main/java/at/tuwien/service/impl/RabbitMqServiceImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..20748480fc50b84e0e99d14ced472da987017d60 --- /dev/null +++ b/fda-query-service/services/src/main/java/at/tuwien/service/impl/RabbitMqServiceImpl.java @@ -0,0 +1,151 @@ +package at.tuwien.service.impl; + +import at.tuwien.api.database.table.TableCsvDto; +import at.tuwien.entities.database.table.Table; +import at.tuwien.exception.*; +import at.tuwien.service.MessageQueueService; +import at.tuwien.service.QueryService; +import at.tuwien.service.TableService; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.rabbitmq.client.*; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.client.HttpClientErrorException; + +import java.io.IOException; +import java.util.HashMap; + +@Log4j2 +@Service +public class RabbitMqServiceImpl implements MessageQueueService { + + private final Channel channel; + private final ObjectMapper objectMapper; + private final QueryService queryService; + private final TableService tableService; + + @Autowired + public RabbitMqServiceImpl(Channel channel, ObjectMapper objectMapper, QueryService queryService, + TableService tableService) { + this.channel = channel; + this.objectMapper = objectMapper; + this.queryService = queryService; + this.tableService = tableService; + } + + @Transactional(readOnly = true) + private void createConsumer(String routingKey, Long containerId, Long databaseId, Long tableId) throws AmqpException { + try { + final String consumerTag = channel.basicConsume(routingKey, true, new Consumer() { + @Override + public void handleConsumeOk(String consumerTag) { + // + } + + @Override + public void handleCancelOk(String consumerTag) { + // + } + + @Override + public void handleCancel(String consumerTag) throws IOException { + // + } + + @Override + public void handleShutdownSignal(String consumerTag, ShutdownSignalException sig) { + // + } + + @Override + public void handleRecoverOk(String consumerTag) { + // + } + + @Override + public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) throws IOException { + final TypeReference<HashMap<String, Object>> payloadReference = new TypeReference<>() { + }; + try { + final TableCsvDto data = TableCsvDto.builder() + .data(objectMapper.readValue(body, payloadReference)) + .build(); + log.debug("received tuple data {}", data); + queryService.insert(containerId, databaseId, tableId, data); + } catch (IOException e) { + log.error("Failed to parse for table with id {}", tableId); + log.debug("Failed to parse for table {} because {}", tableId, e.getMessage()); + /* ignore */ + } catch (HttpClientErrorException.Unauthorized e) { + log.error("Failed to authenticate for table with id {}", tableId); + log.debug("Failed to authenticate for table {} because {}", tableId, e.getMessage()); + /* ignore */ + } catch (HttpClientErrorException.BadRequest e) { + log.error("Failed to insert for table with id {}", tableId); + log.debug("Failed to insert for table {} because {}", tableId, e.getMessage()); + /* ignore */ + } catch (TableNotFoundException e) { + log.error("Failed to find table with id {}", tableId); + /* ignore */ + } catch (TableMalformedException e) { + log.error("Tuple columns do not math table columns with table id {}", tableId); + /* ignore */ + } catch (DatabaseNotFoundException e) { + log.error("Failed to find database with id {}", databaseId); + /* ignore */ + } catch (ImageNotSupportedException e) { + /* ignore */ + } catch (ContainerNotFoundException e) { + log.error("Failed to find container with id {}", containerId); + /* ignore */ + } + } + }); + log.info("Declared consumer for table topic {}", routingKey); + log.debug("declared consumer for table topic {} with tag {}", routingKey, consumerTag); + } catch (IOException e) { + log.error("Failed to create consumer for table with id {}", tableId); + throw new AmqpException("Failed to create consumer", e); + } catch (Exception e) { + log.warn("Failed unknown: {}", e.getMessage()); + /* ignore */ + } + } + + private Boolean hasConsumer(Table table) { + try { + final AMQP.Queue.DeclareOk response = channel.queueDeclarePassive(table.getTopic()); + log.debug("queue {} has {} messages waiting", table.getTopic(), response.getMessageCount()); + log.debug("queue {} has {} consumers", table.getTopic(), response.getConsumerCount()); + return response.getConsumerCount() > 0; + } catch (IOException e) { + log.error("Failed to check if queue {} has consumers", table.getTopic()); + /* ignore */ + } + return false; + } + + @Override + @Transactional(readOnly = true) + @Scheduled(fixedRate = 5000) + public void renewConsumers() { + tableService.findAll() + .forEach(table -> { + if (hasConsumer(table)) { + log.debug("table {} has already one consumer, skipping.", table.getId()); + return; + } + try { + createConsumer(table.getTopic(), table.getDatabase().getContainer() + .getId(), table.getDatabase().getId(), table.getId()); + } catch (AmqpException e) { + /* ignore */ + } + }); + } + +} diff --git a/fda-query-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java b/fda-query-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java index 78a64e57b91eaed69286dd8c4eaf6aaa6aa67c9d..617ed9e7e6e370f60211f956a2581f1690ea463f 100644 --- a/fda-query-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java +++ b/fda-query-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java @@ -39,8 +39,9 @@ public class TableServiceImpl extends HibernateConnector implements TableService @Override @Transactional(readOnly = true) - public Table find(Long databaseId, Long tableId) throws DatabaseNotFoundException, TableNotFoundException { - final Optional<Table> table = tableRepository.findOne(databaseId, tableId); + public Table find(Long containerId, Long databaseId, Long tableId) throws DatabaseNotFoundException, + TableNotFoundException { + final Optional<Table> table = tableRepository.find(containerId, databaseId, tableId); if (table.isEmpty()) { log.error("Failed to find table with id {} of database with id {} in metadata database", tableId, databaseId); @@ -51,7 +52,13 @@ public class TableServiceImpl extends HibernateConnector implements TableService @Override @Transactional(readOnly = true) - public List<TableHistoryDto> findHistory(Long databaseId, Long tableId) + public List<Table> findAll() { + return tableRepository.findAll(); + } + + @Override + @Transactional(readOnly = true) + public List<TableHistoryDto> findHistory(Long containerId, Long databaseId, Long tableId) throws DatabaseNotFoundException, QueryMalformedException, TableNotFoundException { /* find */ final Optional<Database> database = databaseRepository.findById(databaseId); @@ -59,7 +66,7 @@ public class TableServiceImpl extends HibernateConnector implements TableService log.error("Database with id {} not found in metadata database", databaseId); throw new DatabaseNotFoundException("Database not found in metadata database"); } - final Table table = find(databaseId, tableId); + final Table table = find(containerId, databaseId, tableId); /* run query */ final SessionFactory factory = getSessionFactory(database.get(), true); final Session session = factory.openSession(); diff --git a/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java index 70eefb2ac46b9a78d2fc22e9149a5bddd8bdd0d3..ee7b7c6e227935aac290362ee6be0cb4bdd95d08 100644 --- a/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java +++ b/fda-table-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java @@ -65,7 +65,6 @@ public class TableEndpoint { TableNameExistsException, ContainerNotFoundException, UserNotFoundException { final Table table = tableService.createTable(containerId, databaseId, createDto, principal); amqpService.create(table); - amqpService.createConsumer(containerId, databaseId, table); return ResponseEntity.status(HttpStatus.CREATED) .body(tableMapper.tableToTableBriefDto(table)); } diff --git a/fda-table-service/services/src/main/java/at/tuwien/config/AmqpConfig.java b/fda-table-service/services/src/main/java/at/tuwien/config/AmqpConfig.java index 30bc1d4ac59c3dbc8cc34fab514127d30f1c13c5..e9b1381e62228aa2fac455f96efbf20db48bc9e9 100644 --- a/fda-table-service/services/src/main/java/at/tuwien/config/AmqpConfig.java +++ b/fda-table-service/services/src/main/java/at/tuwien/config/AmqpConfig.java @@ -32,7 +32,6 @@ public class AmqpConfig { factory.setHost(ampqHost); factory.setUsername(amqpUsername); factory.setPassword(amqpPassword); - log.debug("AMQP host {}, username {}, password {}", ampqHost, amqpUsername, amqpPassword); final Connection connection = factory.newConnection(); return connection.createChannel(); } diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/MessageQueueService.java b/fda-table-service/services/src/main/java/at/tuwien/service/MessageQueueService.java index 0dfd7a2740df8f8bd83eaaa6d0cca0a5de0705bf..312f03f70c26bb87b23089ba5849cb5131dac5e1 100644 --- a/fda-table-service/services/src/main/java/at/tuwien/service/MessageQueueService.java +++ b/fda-table-service/services/src/main/java/at/tuwien/service/MessageQueueService.java @@ -17,14 +17,4 @@ public interface MessageQueueService { * @throws AmqpException The broker service did not allow to create a consumer. */ void create(Table table) throws AmqpException; - - /** - * Create a consumer - * - * @param containerId - * @param databaseId - * @param table - * @throws AmqpException - */ - void createConsumer(Long containerId, Long databaseId, Table table) throws AmqpException; } diff --git a/fda-table-service/services/src/main/java/at/tuwien/service/impl/RabbitMqService.java b/fda-table-service/services/src/main/java/at/tuwien/service/impl/RabbitMqService.java index ada6316541b3cbe51f03d163d733e7b630b6216f..2602a664868c779db190a95138dac7bcc6e5e6ba 100644 --- a/fda-table-service/services/src/main/java/at/tuwien/service/impl/RabbitMqService.java +++ b/fda-table-service/services/src/main/java/at/tuwien/service/impl/RabbitMqService.java @@ -1,38 +1,25 @@ package at.tuwien.service.impl; -import at.tuwien.api.database.table.TableCsvDto; import at.tuwien.entities.database.table.Table; import at.tuwien.exception.*; -import at.tuwien.gateway.QueryServiceGateway; import at.tuwien.service.MessageQueueService; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.rabbitmq.client.AMQP; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.DefaultConsumer; -import com.rabbitmq.client.Envelope; +import com.rabbitmq.client.*; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import org.springframework.web.client.HttpClientErrorException; import java.io.IOException; -import java.util.HashMap; @Log4j2 @Service public class RabbitMqService implements MessageQueueService { private final Channel channel; - private final ObjectMapper objectMapper; - private final QueryServiceGateway queryServiceGateway; @Autowired - public RabbitMqService(Channel channel, ObjectMapper objectMapper, QueryServiceGateway queryServiceGateway) { + public RabbitMqService(Channel channel) { this.channel = channel; - this.objectMapper = objectMapper; - this.queryServiceGateway = queryServiceGateway; } @Override @@ -50,46 +37,4 @@ public class RabbitMqService implements MessageQueueService { log.debug("created queue for table {}", table); } - @Override - @Transactional(readOnly = true) - public void createConsumer(Long containerId, Long databaseId, Table table) throws AmqpException { - try { - channel.basicConsume(table.getTopic(), true, new DefaultConsumer(channel) { - @Override - public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) { - final TypeReference<HashMap<String, Object>> payloadReference = new TypeReference<>() { - }; - try { - final TableCsvDto data = TableCsvDto.builder() - .data(objectMapper.readValue(body, payloadReference)) - .build(); - log.debug("received tuple data {}", data); - queryServiceGateway.publish(containerId, databaseId, table.getId(), data); - } catch (IOException e) { - log.error("Failed to parse for table with id {}", table.getId()); - log.debug("Failed to parse for table {} because {}", table, e.getMessage()); - /* ignore */ - } catch (HttpClientErrorException.Unauthorized e) { - log.error("Failed to authenticate for table with id {}", table.getId()); - log.debug("Failed to authenticate for table {} because {}", table.getId(), e.getMessage()); - /* ignore */ - } catch (HttpClientErrorException.BadRequest e) { - log.error("Failed to insert for table with id {}", table.getId()); - log.debug("Failed to insert for table {} because {}", table.getId(), e.getMessage()); - /* ignore */ - } - } - }); - } catch (IOException e) { - log.error("Failed to create consumer for table with id {}", table.getId()); - log.debug("Failed to create basic consumer for table {}", table); - throw new AmqpException("Failed to create consumer", e); - } catch (Exception e) { - log.warn("Failed unknown: {}", e.getMessage()); - /* ignore */ - } - log.info("Declared consumer for table topic {}", table.getTopic()); - log.debug("declared consumer {}", table); - } - } diff --git a/fda-ui/components/QueryList.vue b/fda-ui/components/QueryList.vue index f2cb29ce5e6da58bd836294918b8c9c6cf4650cf..ae0c9a11c5dcbcfd999734db3b2915551f0d1de2 100644 --- a/fda-ui/components/QueryList.vue +++ b/fda-ui/components/QueryList.vue @@ -51,7 +51,7 @@ Execution Timestamp </v-list-item-title> <v-list-item-content> - {{ execution }}<br />{{ executionUTC }} + {{ executionUTC }} </v-list-item-content> </v-list-item-content> </v-list-item> @@ -73,7 +73,7 @@ </template> <script> -import { formatTimestamp, formatTimestampUTC } from '@/utils' +import { formatTimestampUTC } from '@/utils' export default { data () { return { @@ -108,9 +108,6 @@ export default { headers: { Authorization: `Bearer ${this.token}` } } }, - execution () { - return formatTimestamp(this.queryDetails.execution) - }, executionUTC () { return formatTimestampUTC(this.queryDetails.execution) } diff --git a/fda-ui/components/TableList.vue b/fda-ui/components/TableList.vue index f0ad9e75fccd01c88913d65dc7bc0421debaad89..85082db335de143688a4bcdf9da46770ad64f168 100644 --- a/fda-ui/components/TableList.vue +++ b/fda-ui/components/TableList.vue @@ -61,7 +61,6 @@ Table Creation </v-list-item-title> <v-list-item-content> - {{ created }}<br /> {{ createdUTC }} </v-list-item-content> </v-list-item-content> @@ -148,7 +147,7 @@ </template> <script> -import { formatTimestamp, formatTimestampUTC } from '@/utils' +import { formatTimestampUTC } from '@/utils' export default { data () { return { @@ -206,9 +205,6 @@ export default { headers: { Authorization: `Bearer ${this.token}` } } }, - created () { - return formatTimestamp(this.tableDetails.created) - }, createdUTC () { return formatTimestampUTC(this.tableDetails.created) } diff --git a/fda-ui/components/TableSchema.vue b/fda-ui/components/TableSchema.vue index fc4b9f43876110377cc464291feafa7b7269c27d..76f80544313c09c3ac9755a5421066604b8ef96b 100644 --- a/fda-ui/components/TableSchema.vue +++ b/fda-ui/components/TableSchema.vue @@ -123,6 +123,7 @@ export default { { value: 'NUMBER', text: 'Number' }, { value: 'BLOB', text: 'Binary Large Object' }, { value: 'DATE', text: 'Date' }, + { value: 'DECIMAL', text: 'Floating Number' }, { value: 'TIMESTAMP', text: 'Timestamp' }, { value: 'DECIMAL', text: 'Decimal' }, { value: 'STRING', text: 'Character Varying' }, diff --git a/fda-ui/components/dialogs/CreateDB.vue b/fda-ui/components/dialogs/CreateDB.vue index 95fbda44b1e0cf5c095e87ea6b75c567a9c95424..e78fec9fc1bedfe650ff884b29d12025a85c0840 100644 --- a/fda-ui/components/dialogs/CreateDB.vue +++ b/fda-ui/components/dialogs/CreateDB.vue @@ -53,7 +53,7 @@ </v-btn> <v-btn id="createDB" - class="mb-2 mr-2 ml-4" + class="mb-2 mr-2" :disabled="!valid || loading" color="primary" type="submit" diff --git a/fda-ui/components/dialogs/EditTuple.vue b/fda-ui/components/dialogs/EditTuple.vue index f2149c7d2d625fbec0d8e5f168907edcf4d51621..3195cb4b1f0a54ef849560436d307f3e79c8fe21 100644 --- a/fda-ui/components/dialogs/EditTuple.vue +++ b/fda-ui/components/dialogs/EditTuple.vue @@ -27,15 +27,7 @@ :label="attr.name" type="text" /> <v-text-field - v-if="attr.column_type === 'DATE'" - v-model="tuple[attr.internal_name]" - suffix="UTC" - class="mb-2" - :required="!attr.is_null_allowed" - :label="attr.name" - type="date" /> - <v-text-field - v-if="attr.column_type === 'TIMESTAMP'" + v-if="attr.column_type === 'TIMESTAMP' || attr.column_type === 'DATE'" v-model="tuple[attr.internal_name]" suffix="UTC" class="mb-2" diff --git a/fda-ui/pages/container/_container_id/database/_database_id/info.vue b/fda-ui/pages/container/_container_id/database/_database_id/info.vue index 64b063eaa38e44b214c3784bde2702debd07e99f..1642d7a2e60a338a75705008af21e2f7ec3ef391 100644 --- a/fda-ui/pages/container/_container_id/database/_database_id/info.vue +++ b/fda-ui/pages/container/_container_id/database/_database_id/info.vue @@ -47,7 +47,6 @@ </v-list-item-title> <v-list-item-content> <v-skeleton-loader v-if="loading" type="text" class="skeleton-small" /> - <span v-if="!loading">{{ created }}</span><br /> <span v-if="!loading">{{ createdUTC }}</span> </v-list-item-content> <v-list-item-title class="mt-2"> @@ -93,7 +92,7 @@ <script> import DBToolbar from '@/components/DBToolbar' import EditDB from '@/components/dialogs/EditDB' -import { formatTimestamp, formatTimestampUTC } from '@/utils' +import { formatTimestampUTC } from '@/utils' export default { components: { @@ -156,9 +155,6 @@ export default { headers: { Authorization: `Bearer ${this.token}` } } }, - created () { - return formatTimestamp(this.database.created) - }, createdUTC () { return formatTimestampUTC(this.database.created) }, diff --git a/fda-ui/pages/container/_container_id/database/_database_id/query/_query_id/index.vue b/fda-ui/pages/container/_container_id/database/_database_id/query/_query_id/index.vue index 200f2244fd7b23abcf6d048b5395cbfa4d9aafe9..aeadd39937265a360fe8c7291d303ad465188692 100644 --- a/fda-ui/pages/container/_container_id/database/_database_id/query/_query_id/index.vue +++ b/fda-ui/pages/container/_container_id/database/_database_id/query/_query_id/index.vue @@ -8,7 +8,7 @@ </v-toolbar-title> <v-spacer /> <v-toolbar-title> - <v-btn v-if="!identifier.id && !loadingIdentifier" color="secondary" class="mr-2" :disabled="!execution || !token" @click.stop="openDialog()"> + <v-btn v-if="!identifier.id && !loadingIdentifier" color="secondary" class="mr-2" :disabled="!executionUTC || !token" @click.stop="openDialog()"> <v-icon left>mdi-content-save-outline</v-icon> Save </v-btn> <v-btn v-if="result_visibility" color="primary" :loading="downloadLoading" @click.stop="download"> @@ -38,10 +38,10 @@ <span v-if="!loadingDatabase && database_visibility">Public</span> <span v-if="!loadingDatabase && !database_visibility">Private</span> </v-list-item-content> - <v-list-item-title class="mt-2"> + <v-list-item-title v-if="database.name" class="mt-2"> Database Name </v-list-item-title> - <v-list-item-content> + <v-list-item-content v-if="database.name"> <v-skeleton-loader v-if="loadingDatabase" type="text" class="skeleton-small" /> <span v-if="!loadingDatabase">{{ database.name }}</span> </v-list-item-content> @@ -52,10 +52,10 @@ <v-skeleton-loader v-if="loadingDatabase" type="text" class="skeleton-small" /> <span v-if="!loadingDatabase">{{ database.publisher }}</span> </v-list-item-content> - <v-list-item-title v-if="database.license" class="mt-2"> + <v-list-item-title v-if="database.license.identifier" class="mt-2"> Database License </v-list-item-title> - <v-list-item-content v-if="database.license"> + <v-list-item-content v-if="database.license.identifier"> <v-skeleton-loader v-if="loadingDatabase" type="text" class="skeleton-xsmall" /> <a v-if="!loadingDatabase" :href="database.license.uri">{{ database.license.identifier }}</a> </v-list-item-content> @@ -138,7 +138,7 @@ </v-list-item-title> <v-list-item-content> <v-skeleton-loader v-if="loadingQuery" type="text" class="skeleton-large" /> - <pre v-if="!loadingQuery">{{ query.query }}</pre> + <pre v-if="!loadingQuery">{{ query_statement }}</pre> </v-list-item-content> <v-list-item-title class="mt-2"> Subset Hash @@ -164,7 +164,6 @@ </v-list-item-title> <v-list-item-content> <v-skeleton-loader v-if="loadingQuery" type="text" class="skeleton-small" /> - <span v-if="!loadingQuery">{{ execution }}</span><br /> <span v-if="!loadingQuery">{{ executionUTC }}</span> </v-list-item-content> </v-list-item-content> @@ -215,7 +214,7 @@ <script> import PersistQuery from '@/components/dialogs/PersistQuery' import OrcidIcon from '@/components/icons/OrcidIcon' -import { formatTimestamp, formatTimestampUTC } from '@/utils' +import { formatTimestampUTC } from '@/utils' export default { name: 'QueryShow', @@ -306,6 +305,9 @@ export default { headers: { Authorization: `Bearer ${this.token}` } } }, + query_statement () { + return this.query.query ? this.query.query : this.identifier.query + }, publisher () { if (this.database.publisher === null) { return 'NA' @@ -342,9 +344,6 @@ export default { result_hash () { return 'sha256:' + (this.identifier.id ? this.identifier.result_hash : this.query.result_hash) }, - execution () { - return this.identifier.id ? formatTimestamp(this.identifier.execution) : formatTimestamp(this.query.execution) - }, executionUTC () { return this.identifier.id ? formatTimestampUTC(this.identifier.execution) : formatTimestampUTC(this.query.execution) }, @@ -431,10 +430,6 @@ export default { console.debug('database', res.data) this.database = res.data } catch (err) { - if (err.response.status !== 401 && err.response.status !== 405) { - console.error('Could not load database', err) - this.$toast.error('Could not load database') - } this.error = true } this.loadingDatabase = false diff --git a/fda-ui/pages/container/index.vue b/fda-ui/pages/container/index.vue index 03215bb8526e8af370c1bfe17b6903f0e9e42224..e4c4c60d17c8c2950139165c0d2edf7bf28b0bea 100644 --- a/fda-ui/pages/container/index.vue +++ b/fda-ui/pages/container/index.vue @@ -46,7 +46,7 @@ <v-icon v-if="item.database.creator.email_verified" small color="primary">mdi-check-decagram</v-icon> </sup> </td> - <td>{{ created(item.created) }}<br/>{{ createdUTC(item.created) }}</td> + <td>{{ createdUTC(item.created) }}</td> </tr> </tbody> </template> @@ -64,7 +64,7 @@ <script> import { mdiDatabaseArrowRightOutline } from '@mdi/js' import CreateDB from '@/components/dialogs/CreateDB' -import { formatTimestamp, formatTimestampUTC } from '@/utils' +import { formatTimestampUTC } from '@/utils' export default { components: { @@ -153,9 +153,6 @@ export default { this.containers = containers console.debug('databases loaded', this.containers) }, - created (str) { - return formatTimestamp(str) - }, createdUTC (str) { return formatTimestampUTC(str) }, diff --git a/fda-ui/pages/login.vue b/fda-ui/pages/login.vue index 38a880108cdb7515a8644f916c3fd3315dace029..6a7eb5909694758efc64ffcdb372209bac96a986 100644 --- a/fda-ui/pages/login.vue +++ b/fda-ui/pages/login.vue @@ -99,14 +99,21 @@ export default { this.$toast.success('Welcome back!') this.$router.push('/container') } catch (err) { - if (err.response !== undefined && err.response.status !== undefined && err.response.status === 418) { - this.$toast.error('Check your inbox and confirm your e-mail address. Login not successful.') - console.error('user has not confirmed e-mail', err) - this.loading = false - return + if (err.response !== undefined && err.response.status !== undefined) { + if (err.response.status === 418) { + this.$toast.error('Check your inbox and confirm your e-mail address.') + console.error('user has not confirmed e-mail', err) + this.loading = false + return + } else if (err.response.status === 404) { + this.$toast.error('Username not found.') + console.error('user has not confirmed e-mail', err) + this.loading = false + return + } + console.error('login user failed', err) + this.$toast.error('Login not successful.') } - console.error('login user failed', err) - this.$toast.error('Login not successful.') } this.loading = false }, diff --git a/fda-ui/server-middleware/index.js b/fda-ui/server-middleware/index.js index ee55f841cd59a37a1d580bfab9de631620155b3d..39489ec6aeebba613a7ee6fc7001940e6667126f 100644 --- a/fda-ui/server-middleware/index.js +++ b/fda-ui/server-middleware/index.js @@ -13,7 +13,7 @@ const colTypeMap = { Boolean: 'BOOLEAN', Date: 'DATE', Integer: 'NUMBER', - Numeric: 'NUMBER', + Decimal: 'DECIMAL', String: 'STRING', Text: 'STRING', Timestamp: 'TIMESTAMP' diff --git a/fda-ui/utils/index.js b/fda-ui/utils/index.js index 4307fa12e708c587bc36d01bbbe39ebcf0c72ba5..5359583a7336f44ecce1f08e145a5b9082126186 100644 --- a/fda-ui/utils/index.js +++ b/fda-ui/utils/index.js @@ -40,7 +40,7 @@ function formatTimestamp (str) { if (str === null) { return null } - return format(new Date(str), 'dd.MM.yyyy HH:mm:ss') + return format(new Date(str), 'yyyy-MM-dd HH:mm:ss') } function formatTimestampUTC (str) { @@ -48,7 +48,7 @@ function formatTimestampUTC (str) { return null } const date = new Date(str).toISOString().slice(0, -1) - return format(new Date(date), 'dd.MM.yyyy HH:mm:ss') + ' (UTC)' + return format(new Date(date), 'yyyy-MM-dd HH:mm:ss') + ' (UTC)' } module.exports = {