diff --git a/.gitignore b/.gitignore
index 716e25c58a793ef0a1470c3dc924baa6a2479c79..9cf181a38e0d982c8da5a8dfab71c6257c6a1c28 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,7 +21,6 @@ final/
.$*
# Notebooks
-.jupyter/
.pytest_cache/
__pycache__/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 08b06743451c8f037a660d9f265506c45bba9895..647d508a1d1631626df1419087760771a7ca59c2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -583,7 +583,7 @@ release-libs:
image: docker.io/python:3.11-alpine
only:
refs:
- - /^release-[0-9]+.*/
+ - /^release-.*/
variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
script:
@@ -591,8 +591,5 @@ release-libs:
- pip install pipenv
- pipenv install gunicorn && pipenv install --dev --system --deploy
- pip install twine build
- - 'sed -i -e "s/__APPVERSION__/${APP_VERSION}/g" ./lib/python/pyproject.toml ./lib/python/setup.py ./lib/python/README.md'
- - python -m build --sdist ./lib/python
- - python -m build --wheel ./lib/python
- - printf "[diskutils]\nindex-servers =\n pypi\n\n[pypi]\nusername = __token__\npassword = ${CI_PIPY_TOKEN}\nrepository = https://upload.pypi.org/legacy/" > .pypirc
- - python -m twine upload --config-file .pypirc --verbose --repository pypi ./lib/python/dist/dbrepo-*
\ No newline at end of file
+ - ./lib/python/package.sh
+ - ./lib/python/release.sh
\ No newline at end of file
diff --git a/.jupyter/.env b/.jupyter/.env
new file mode 100644
index 0000000000000000000000000000000000000000..119dca696afda5f423ce2605cfaa41d8c84666c1
--- /dev/null
+++ b/.jupyter/.env
@@ -0,0 +1,3 @@
+DBREPO_ENDPOINT=https://test.dbrepo.tuwien.ac.at
+DBREPO_USERNAME=foo
+DBREPO_PASSWORD=bar
diff --git a/.jupyter/.gitignore b/.jupyter/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..a5aee567d469985ea55322421c496814c680017f
--- /dev/null
+++ b/.jupyter/.gitignore
@@ -0,0 +1,2 @@
+# environment
+venv/
\ No newline at end of file
diff --git a/.jupyter/default.ipynb b/.jupyter/default.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..0d51aa7188a956b2256a59c0110f9f7d6357cf3c
--- /dev/null
+++ b/.jupyter/default.ipynb
@@ -0,0 +1,99 @@
+{
+ "cells": [
+ {
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2024-04-10T06:10:08.974108Z",
+ "start_time": "2024-04-10T06:10:07.365075Z"
+ }
+ },
+ "cell_type": "code",
+ "source": [
+ "!pip install python-dotenv dbrepo==1.4.2rc10\n",
+ "import dotenv\n",
+ "%load_ext dotenv\n",
+ "%dotenv"
+ ],
+ "id": "4eb6c2470f464173",
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: python-dotenv in ./venv/lib/python3.11/site-packages (1.0.1)\r\n",
+ "Requirement already satisfied: dbrepo==1.4.2rc10 in ./venv/lib/python3.11/site-packages (1.4.2rc10)\r\n",
+ "Requirement already satisfied: requests>=2.31 in ./venv/lib/python3.11/site-packages (from dbrepo==1.4.2rc10) (2.31.0)\r\n",
+ "Requirement already satisfied: pika in ./venv/lib/python3.11/site-packages (from dbrepo==1.4.2rc10) (1.3.2)\r\n",
+ "Requirement already satisfied: pydantic in ./venv/lib/python3.11/site-packages (from dbrepo==1.4.2rc10) (2.6.4)\r\n",
+ "Requirement already satisfied: tuspy in ./venv/lib/python3.11/site-packages (from dbrepo==1.4.2rc10) (1.0.3)\r\n",
+ "Requirement already satisfied: charset-normalizer<4,>=2 in ./venv/lib/python3.11/site-packages (from requests>=2.31->dbrepo==1.4.2rc10) (3.3.2)\r\n",
+ "Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.11/site-packages (from requests>=2.31->dbrepo==1.4.2rc10) (3.6)\r\n",
+ "Requirement already satisfied: urllib3<3,>=1.21.1 in ./venv/lib/python3.11/site-packages (from requests>=2.31->dbrepo==1.4.2rc10) (2.2.1)\r\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.11/site-packages (from requests>=2.31->dbrepo==1.4.2rc10) (2024.2.2)\r\n",
+ "Requirement already satisfied: annotated-types>=0.4.0 in ./venv/lib/python3.11/site-packages (from pydantic->dbrepo==1.4.2rc10) (0.6.0)\r\n",
+ "Requirement already satisfied: pydantic-core==2.16.3 in ./venv/lib/python3.11/site-packages (from pydantic->dbrepo==1.4.2rc10) (2.16.3)\r\n",
+ "Requirement already satisfied: typing-extensions>=4.6.1 in ./venv/lib/python3.11/site-packages (from pydantic->dbrepo==1.4.2rc10) (4.11.0)\r\n",
+ "Requirement already satisfied: tinydb>=3.5.0 in ./venv/lib/python3.11/site-packages (from tuspy->dbrepo==1.4.2rc10) (4.8.0)\r\n",
+ "Requirement already satisfied: aiohttp>=3.6.2 in ./venv/lib/python3.11/site-packages (from tuspy->dbrepo==1.4.2rc10) (3.9.3)\r\n",
+ "Requirement already satisfied: aiosignal>=1.1.2 in ./venv/lib/python3.11/site-packages (from aiohttp>=3.6.2->tuspy->dbrepo==1.4.2rc10) (1.3.1)\r\n",
+ "Requirement already satisfied: attrs>=17.3.0 in ./venv/lib/python3.11/site-packages (from aiohttp>=3.6.2->tuspy->dbrepo==1.4.2rc10) (23.2.0)\r\n",
+ "Requirement already satisfied: frozenlist>=1.1.1 in ./venv/lib/python3.11/site-packages (from aiohttp>=3.6.2->tuspy->dbrepo==1.4.2rc10) (1.4.1)\r\n",
+ "Requirement already satisfied: multidict<7.0,>=4.5 in ./venv/lib/python3.11/site-packages (from aiohttp>=3.6.2->tuspy->dbrepo==1.4.2rc10) (6.0.5)\r\n",
+ "Requirement already satisfied: yarl<2.0,>=1.0 in ./venv/lib/python3.11/site-packages (from aiohttp>=3.6.2->tuspy->dbrepo==1.4.2rc10) (1.9.4)\r\n"
+ ]
+ }
+ ],
+ "execution_count": 1
+ },
+ {
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2024-04-10T06:10:09.860353Z",
+ "start_time": "2024-04-10T06:10:08.981192Z"
+ }
+ },
+ "cell_type": "code",
+ "source": [
+ "from dbrepo.RestClient import RestClient\n",
+ "client = RestClient()\n",
+ "analysis = client.get_licenses()"
+ ],
+ "id": "initial_id",
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2024-04-10 08:10:09,327 root DEBUG method: get\n",
+ "2024-04-10 08:10:09,328 root DEBUG url: https://test.dbrepo.tuwien.ac.at/api/database/license\n",
+ "2024-04-10 08:10:09,328 root DEBUG stream: False\n",
+ "2024-04-10 08:10:09,329 root DEBUG secure: True\n",
+ "2024-04-10 08:10:09,329 root DEBUG username: foo, password: (hidden)\n"
+ ]
+ }
+ ],
+ "execution_count": 2
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.6"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/.jupyter/requirements.txt b/.jupyter/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c06c064500982d0b62a37283aeaac06c85ddb91b
--- /dev/null
+++ b/.jupyter/requirements.txt
@@ -0,0 +1,3 @@
+python-dotenv==1.0.1
+notebook==7.1.2
+dbrepo==1.4.2rc10
diff --git a/bin/test.sh b/bin/test.sh
deleted file mode 100755
index 765ecaf6b7e2741a5e18d33f29071309d053de16..0000000000000000000000000000000000000000
--- a/bin/test.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-mvn -f ./dbrepo-metadata-service/pom.xml clean install -DskipTests
-# test java services
-mvn -f ./dbrepo-metadata-service/pom.xml clean test verify
-mvn -f ./dbrepo-data-service/pom.xml clean test verify
-# test python services
-bash ./dbrepo-analyse-service/test.sh
-bash ./dbrepo-search-service/test.sh
-# test ui
-yarn --cwd ./dbrepo-ui install
-yarn --cwd ./dbrepo-ui run test:unit
-yarn --cwd ./dbrepo-ui run coverage
\ No newline at end of file
diff --git a/lib/python/Pipfile b/lib/python/Pipfile
index a6f3abd4b319a9e9aff95ddc2f3194106eff11b5..b52daf9ec43934af8590c55697672ae74cb1c2f2 100644
--- a/lib/python/Pipfile
+++ b/lib/python/Pipfile
@@ -8,6 +8,7 @@ requests = "~=2.31"
pika = "*"
pydantic = "*"
tuspy = "*"
+pandas = "*"
[dev-packages]
build = "*"
diff --git a/lib/python/Pipfile.lock b/lib/python/Pipfile.lock
index d1c478e1745a28f92bc20c94d2b145ad79b85bb2..554a33747d2a6ae6e9e80a30b90228bdf4d98ad2 100644
--- a/lib/python/Pipfile.lock
+++ b/lib/python/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "8cc220d86727c0bc277f084e48a33c5a487e40639ac8ac36ef8f5d824784261d"
+ "sha256": "075c662404a333e1251df1222db0602a779feb803e123e80df47260cb2e4ebf9"
},
"pipfile-spec": 6,
"requires": {
@@ -413,6 +413,84 @@
"markers": "python_version >= '3.7'",
"version": "==6.0.5"
},
+ "numpy": {
+ "hashes": [
+ "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b",
+ "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818",
+ "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20",
+ "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0",
+ "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010",
+ "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a",
+ "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea",
+ "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c",
+ "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71",
+ "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110",
+ "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be",
+ "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a",
+ "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a",
+ "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5",
+ "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed",
+ "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd",
+ "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c",
+ "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e",
+ "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0",
+ "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c",
+ "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a",
+ "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b",
+ "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0",
+ "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6",
+ "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2",
+ "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a",
+ "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30",
+ "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218",
+ "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5",
+ "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07",
+ "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2",
+ "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4",
+ "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764",
+ "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef",
+ "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3",
+ "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"
+ ],
+ "markers": "python_version == '3.11'",
+ "version": "==1.26.4"
+ },
+ "pandas": {
+ "hashes": [
+ "sha256:04f6ec3baec203c13e3f8b139fb0f9f86cd8c0b94603ae3ae8ce9a422e9f5bee",
+ "sha256:06cf591dbaefb6da9de8472535b185cba556d0ce2e6ed28e21d919704fef1a9e",
+ "sha256:0ab90f87093c13f3e8fa45b48ba9f39181046e8f3317d3aadb2fffbb1b978572",
+ "sha256:0f573ab277252ed9aaf38240f3b54cfc90fff8e5cab70411ee1d03f5d51f3944",
+ "sha256:101d0eb9c5361aa0146f500773395a03839a5e6ecde4d4b6ced88b7e5a1a6403",
+ "sha256:11940e9e3056576ac3244baef2fedade891977bcc1cb7e5cc8f8cc7d603edc89",
+ "sha256:1ba21b1d5c0e43416218db63037dbe1a01fc101dc6e6024bcad08123e48004ab",
+ "sha256:4aa1d8707812a658debf03824016bf5ea0d516afdea29b7dc14cf687bc4d4ec6",
+ "sha256:4acf681325ee1c7f950d058b05a820441075b0dd9a2adf5c4835b9bc056bf4fb",
+ "sha256:53680dc9b2519cbf609c62db3ed7c0b499077c7fefda564e330286e619ff0dd9",
+ "sha256:739cc70eaf17d57608639e74d63387b0d8594ce02f69e7a0b046f117974b3019",
+ "sha256:76f27a809cda87e07f192f001d11adc2b930e93a2b0c4a236fde5429527423be",
+ "sha256:7d2ed41c319c9fb4fd454fe25372028dfa417aacb9790f68171b2e3f06eae8cd",
+ "sha256:88ecb5c01bb9ca927ebc4098136038519aa5d66b44671861ffab754cae75102c",
+ "sha256:8df8612be9cd1c7797c93e1c5df861b2ddda0b48b08f2c3eaa0702cf88fb5f88",
+ "sha256:94e714a1cca63e4f5939cdce5f29ba8d415d85166be3441165edd427dc9f6bc0",
+ "sha256:9bd8a40f47080825af4317d0340c656744f2bfdb6819f818e6ba3cd24c0e1397",
+ "sha256:9d1265545f579edf3f8f0cb6f89f234f5e44ba725a34d86535b1a1d38decbccc",
+ "sha256:a935a90a76c44fe170d01e90a3594beef9e9a6220021acfb26053d01426f7dc2",
+ "sha256:af5d3c00557d657c8773ef9ee702c61dd13b9d7426794c9dfeb1dc4a0bf0ebc7",
+ "sha256:c2ce852e1cf2509a69e98358e8458775f89599566ac3775e70419b98615f4b06",
+ "sha256:c38ce92cb22a4bea4e3929429aa1067a454dcc9c335799af93ba9be21b6beb51",
+ "sha256:c391f594aae2fd9f679d419e9a4d5ba4bce5bb13f6a989195656e7dc4b95c8f0",
+ "sha256:c70e00c2d894cb230e5c15e4b1e1e6b2b478e09cf27cc593a11ef955b9ecc81a",
+ "sha256:df0c37ebd19e11d089ceba66eba59a168242fc6b7155cba4ffffa6eccdfb8f16",
+ "sha256:e97fbb5387c69209f134893abc788a6486dbf2f9e511070ca05eed4b930b1b02",
+ "sha256:f02a3a6c83df4026e55b63c1f06476c9aa3ed6af3d89b4f04ea656ccdaaaa359",
+ "sha256:f821213d48f4ab353d20ebc24e4faf94ba40d76680642fb7ce2ea31a3ad94f9b",
+ "sha256:f9d3558d263073ed95e46f4650becff0c5e1ffe0fc3a015de3c79283dfbdb3df"
+ ],
+ "index": "pypi",
+ "markers": "python_version >= '3.9'",
+ "version": "==2.2.1"
+ },
"pika": {
"hashes": [
"sha256:0779a7c1fafd805672796085560d290213a465e4f6f76a6fb19e378d8041a14f",
@@ -516,6 +594,21 @@
"markers": "python_version >= '3.8'",
"version": "==2.16.3"
},
+ "python-dateutil": {
+ "hashes": [
+ "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
+ "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==2.9.0.post0"
+ },
+ "pytz": {
+ "hashes": [
+ "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812",
+ "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"
+ ],
+ "version": "==2024.1"
+ },
"requests": {
"hashes": [
"sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
@@ -525,6 +618,14 @@
"markers": "python_version >= '3.7'",
"version": "==2.31.0"
},
+ "six": {
+ "hashes": [
+ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+ "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+ ],
+ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+ "version": "==1.16.0"
+ },
"tinydb": {
"hashes": [
"sha256:30c06d12383d7c332e404ca6a6103fb2b32cbf25712689648c39d9a6bd34bd3d",
@@ -544,11 +645,19 @@
},
"typing-extensions": {
"hashes": [
- "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475",
- "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"
+ "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0",
+ "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"
],
"markers": "python_version >= '3.8'",
- "version": "==4.10.0"
+ "version": "==4.11.0"
+ },
+ "tzdata": {
+ "hashes": [
+ "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd",
+ "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"
+ ],
+ "markers": "python_version >= '2'",
+ "version": "==2024.1"
},
"urllib3": {
"hashes": [
@@ -672,6 +781,14 @@
"markers": "python_version >= '3.7'",
"version": "==2.14.0"
},
+ "backports.tarfile": {
+ "hashes": [
+ "sha256:2688f159c21afd56a07b75f01306f9f52c79aebcc5f4a117fb8fbb4445352c75",
+ "sha256:bcd36290d9684beb524d3fe74f4a2db056824c47746583f090b8e55daf0776e4"
+ ],
+ "markers": "python_version < '3.12'",
+ "version": "==1.0.0"
+ },
"beautifulsoup4": {
"hashes": [
"sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051",
@@ -682,12 +799,12 @@
},
"build": {
"hashes": [
- "sha256:8ed0851ee76e6e38adce47e4bee3b51c771d86c64cf578d0c2245567ee200e73",
- "sha256:8eea65bb45b1aac2e734ba2cc8dad3a6d97d97901a395bd0ed3e7b46953d2a31"
+ "sha256:526263f4870c26f26c433545579475377b2b7588b6f1eac76a001e873ae3e19d",
+ "sha256:75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4"
],
"index": "pypi",
- "markers": "python_version >= '3.7'",
- "version": "==1.1.1"
+ "markers": "python_version >= '3.8'",
+ "version": "==1.2.1"
},
"certifi": {
"hashes": [
@@ -999,19 +1116,19 @@
},
"jaraco.classes": {
"hashes": [
- "sha256:86b534de565381f6b3c1c830d13f931d7be1a75f0081c57dff615578676e2206",
- "sha256:cb28a5ebda8bc47d8c8015307d93163464f9f2b91ab4006e09ff0ce07e8bfb30"
+ "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd",
+ "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790"
],
"markers": "python_version >= '3.8'",
- "version": "==3.3.1"
+ "version": "==3.4.0"
},
"jaraco.context": {
"hashes": [
- "sha256:4dad2404540b936a20acedec53355bdaea223acb88fd329fa6de9261c941566e",
- "sha256:5d9e95ca0faa78943ed66f6bc658dd637430f16125d86988e77844c741ff2f11"
+ "sha256:3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266",
+ "sha256:c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2"
],
- "markers": "python_version >= '3.7'",
- "version": "==4.3.0"
+ "markers": "python_version >= '3.8'",
+ "version": "==5.3.0"
},
"jaraco.functools": {
"hashes": [
@@ -1039,11 +1156,11 @@
},
"keyring": {
"hashes": [
- "sha256:9a15cd280338920388e8c1787cb8792b9755dabb3e7c61af5ac1f8cd437cefde",
- "sha256:fc024ed53c7ea090e30723e6bd82f58a39dc25d9a6797d866203ecd0ee6306cb"
+ "sha256:26fc12e6a329d61d24aa47b22a7c5c3f35753df7d8f2860973cf94f4e1fb3427",
+ "sha256:7230ea690525133f6ad536a9b5def74a4bd52642abe594761028fc044d7c7893"
],
"markers": "python_version >= '3.8'",
- "version": "==25.0.0"
+ "version": "==25.1.0"
},
"markdown-it-py": {
"hashes": [
@@ -1137,24 +1254,24 @@
},
"nh3": {
"hashes": [
- "sha256:0d02d0ff79dfd8208ed25a39c12cbda092388fff7f1662466e27d97ad011b770",
- "sha256:3277481293b868b2715907310c7be0f1b9d10491d5adf9fce11756a97e97eddf",
- "sha256:3b803a5875e7234907f7d64777dfde2b93db992376f3d6d7af7f3bc347deb305",
- "sha256:427fecbb1031db085eaac9931362adf4a796428ef0163070c484b5a768e71601",
- "sha256:5f0d77272ce6d34db6c87b4f894f037d55183d9518f948bba236fe81e2bb4e28",
- "sha256:60684857cfa8fdbb74daa867e5cad3f0c9789415aba660614fe16cd66cbb9ec7",
- "sha256:6f42f99f0cf6312e470b6c09e04da31f9abaadcd3eb591d7d1a88ea931dca7f3",
- "sha256:86e447a63ca0b16318deb62498db4f76fc60699ce0a1231262880b38b6cff911",
- "sha256:8d595df02413aa38586c24811237e95937ef18304e108b7e92c890a06793e3bf",
- "sha256:9c0d415f6b7f2338f93035bba5c0d8c1b464e538bfbb1d598acd47d7969284f0",
- "sha256:a5167a6403d19c515217b6bcaaa9be420974a6ac30e0da9e84d4fc67a5d474c5",
- "sha256:ac19c0d68cd42ecd7ead91a3a032fdfff23d29302dbb1311e641a130dfefba97",
- "sha256:b1e97221cedaf15a54f5243f2c5894bb12ca951ae4ddfd02a9d4ea9df9e1a29d",
- "sha256:bc2d086fb540d0fa52ce35afaded4ea526b8fc4d3339f783db55c95de40ef02e",
- "sha256:d1e30ff2d8d58fb2a14961f7aac1bbb1c51f9bdd7da727be35c63826060b0bf3",
- "sha256:f3b53ba93bb7725acab1e030bc2ecd012a817040fd7851b332f86e2f9bb98dc6"
- ],
- "version": "==0.2.15"
+ "sha256:0316c25b76289cf23be6b66c77d3608a4fdf537b35426280032f432f14291b9a",
+ "sha256:1a814dd7bba1cb0aba5bcb9bebcc88fd801b63e21e2450ae6c52d3b3336bc911",
+ "sha256:1aa52a7def528297f256de0844e8dd680ee279e79583c76d6fa73a978186ddfb",
+ "sha256:22c26e20acbb253a5bdd33d432a326d18508a910e4dcf9a3316179860d53345a",
+ "sha256:40015514022af31975c0b3bca4014634fa13cb5dc4dbcbc00570acc781316dcc",
+ "sha256:40d0741a19c3d645e54efba71cb0d8c475b59135c1e3c580f879ad5514cbf028",
+ "sha256:551672fd71d06cd828e282abdb810d1be24e1abb7ae2543a8fa36a71c1006fe9",
+ "sha256:66f17d78826096291bd264f260213d2b3905e3c7fae6dfc5337d49429f1dc9f3",
+ "sha256:85cdbcca8ef10733bd31f931956f7fbb85145a4d11ab9e6742bbf44d88b7e351",
+ "sha256:a3f55fabe29164ba6026b5ad5c3151c314d136fd67415a17660b4aaddacf1b10",
+ "sha256:b4427ef0d2dfdec10b641ed0bdaf17957eb625b2ec0ea9329b3d28806c153d71",
+ "sha256:ba73a2f8d3a1b966e9cdba7b211779ad8a2561d2dba9674b8a19ed817923f65f",
+ "sha256:c21bac1a7245cbd88c0b0e4a420221b7bfa838a2814ee5bb924e9c2f10a1120b",
+ "sha256:c551eb2a3876e8ff2ac63dff1585236ed5dfec5ffd82216a7a174f7c5082a78a",
+ "sha256:c790769152308421283679a142dbdb3d1c46c79c823008ecea8e8141db1a2062",
+ "sha256:d7a25fd8c86657f5d9d576268e3b3767c5cd4f42867c9383618be8517f0f022a"
+ ],
+ "version": "==0.2.17"
},
"packaging": {
"hashes": [
@@ -1182,10 +1299,11 @@
},
"pycparser": {
"hashes": [
- "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9",
- "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"
+ "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6",
+ "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"
],
- "version": "==2.21"
+ "markers": "python_version >= '3.8'",
+ "version": "==2.22"
},
"pygments": {
"hashes": [
@@ -1231,11 +1349,12 @@
},
"requests-mock": {
"hashes": [
- "sha256:ef10b572b489a5f28e09b708697208c4a3b2b89ef80a9f01584340ea357ec3c4",
- "sha256:f7fae383f228633f6bececebdab236c478ace2284d6292c6e7e2867b9ab74d15"
+ "sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563",
+ "sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401"
],
"index": "pypi",
- "version": "==1.11.0"
+ "markers": "python_version >= '3.5'",
+ "version": "==1.12.1"
},
"requests-toolbelt": {
"hashes": [
@@ -1278,14 +1397,6 @@
"markers": "python_version >= '3.8'",
"version": "==69.2.0"
},
- "six": {
- "hashes": [
- "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
- "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
- ],
- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
- "version": "==1.16.0"
- },
"snowballstemmer": {
"hashes": [
"sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1",
diff --git a/lib/python/README.md b/lib/python/README.md
index 0cb742a451c6b6e9cb8e44bdc3b5068fd3d5d2b3..610c6eded42360330561c723ef1283d075ec6b2c 100644
--- a/lib/python/README.md
+++ b/lib/python/README.md
@@ -32,33 +32,33 @@ print(f"Analysis result: {analysis}")
# create table
table = client.create_table(database_id=1,
- name="Sensor Data",
- constraints=CreateTableConstraints(
- checks=['precipitation >= 0'],
- uniques=[['precipitation']]),
- columns=[CreateTableColumn(name="date",
- type=ColumnType.DATE,
- dfid=3, # YYYY-MM-dd
- primary_key=True,
- null_allowed=False),
- CreateTableColumn(name="precipitation",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True),
- CreateTableColumn(name="lat",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True),
- CreateTableColumn(name="lng",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True)])
+ name="Sensor Data",
+ constraints=CreateTableConstraints(
+ checks=['precipitation >= 0'],
+ uniques=[['precipitation']]),
+ columns=[CreateTableColumn(name="date",
+ type=ColumnType.DATE,
+ dfid=3, # YYYY-MM-dd
+ primary_key=True,
+ null_allowed=False),
+ CreateTableColumn(name="precipitation",
+ type=ColumnType.DECIMAL,
+ size=10,
+ d=4,
+ primary_key=False,
+ null_allowed=True),
+ CreateTableColumn(name="lat",
+ type=ColumnType.DECIMAL,
+ size=10,
+ d=4,
+ primary_key=False,
+ null_allowed=True),
+ CreateTableColumn(name="lng",
+ type=ColumnType.DECIMAL,
+ size=10,
+ d=4,
+ primary_key=False,
+ null_allowed=True)])
print(f"Create table result {table}")
# -> (id=1, internal_name=sensor_data, ...)
@@ -83,7 +83,25 @@ print(f"Finished.")
queries ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//usage-overview/#export-subset))
- Get data from tables/views/subsets
-## Future
+## Configure
+
+All credentials can optionally be set/overridden with environment variables. This is especially useful when sharing
+Jupyter Notebooks by creating an invisible `.env` file and loading it:
+
+```
+REST_API_ENDPOINT="https://test.dbrepo.tuwien.ac.at"
+REST_API_USERNAME="foo"
+REST_API_PASSWORD="bar"
+REST_API_SECURE="True"
+AMQP_API_HOST="https://test.dbrepo.tuwien.ac.at"
+AMQP_API_PORT="5672"
+AMQP_API_USERNAME="foo"
+AMQP_API_PASSWORD="bar"
+AMQP_API_VIRTUAL_HOST="/"
+REST_UPLOAD_ENDPOINT="https://test.dbrepo.tuwien.ac.at/api/upload/files"
+```
+
+## Roadmap
- Searching
diff --git a/lib/python/build.sh b/lib/python/build.sh
deleted file mode 100644
index 802067b6beb9c11b6f34f703a2f1a8257fba5d3a..0000000000000000000000000000000000000000
--- a/lib/python/build.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-# needed for MariaDB Connector/C
-apt update && apt install -y curl gcc libmariadb3 libmariadb-dev
-
-python3 -m venv ./lib/python/venv
-source ./lib/python/venv/bin/activate
-PIPENV_PIPFILE=./lib/python/Pipfile pipenv install --dev
\ No newline at end of file
diff --git a/lib/python/dbrepo/AmqpClient.py b/lib/python/dbrepo/AmqpClient.py
index c9fcdc05ffe9116083c42c24b73242c80c43e431..29f7e261ec638eff1f7d2287219f4067ff30af65 100644
--- a/lib/python/dbrepo/AmqpClient.py
+++ b/lib/python/dbrepo/AmqpClient.py
@@ -2,8 +2,8 @@ import dataclasses
import os
import pika
import sys
-import logging
import json
+import logging
from dbrepo.api.dto import CreateData
@@ -32,16 +32,18 @@ class AmqpClient:
broker_virtual_host: str = '/',
username: str = None,
password: str = None) -> None:
+ logging.getLogger('requests').setLevel(logging.INFO)
+ logging.getLogger('urllib3').setLevel(logging.INFO)
logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-6s %(message)s', level=logging.DEBUG,
stream=sys.stdout)
- self.broker_host = os.environ.get('DBREPO_BROKER_HOST', broker_host)
- self.broker_port = os.environ.get('DBREPO_BROKER_PORT', broker_port)
- if os.environ.get('DBREPO_BROKER_VIRTUAL_HOST') is not None:
- self.broker_virtual_host = os.environ.get('DBREPO_BROKER_VIRTUAL_HOST')
+ self.broker_host = os.environ.get('AMQP_API_HOST', broker_host)
+ self.broker_port = os.environ.get('AMQP_API_PORT', broker_port)
+ if os.environ.get('AMQP_API_VIRTUAL_HOST') is not None:
+ self.broker_virtual_host = os.environ.get('AMQP_API_VIRTUAL_HOST')
else:
self.broker_virtual_host = broker_virtual_host
- self.username = os.environ.get('DBREPO_USERNAME', username)
- self.password = os.environ.get('DBREPO_PASSWORD', password)
+ self.username = os.environ.get('AMQP_API_USERNAME', username)
+ self.password = os.environ.get('AMQP_API_PASSWORD', password)
def publish(self, exchange: str, routing_key: str, data=dict) -> None:
"""
diff --git a/lib/python/dbrepo/RestClient.py b/lib/python/dbrepo/RestClient.py
index a114dcc9889fc7fa81de30ff716082a905d2c4eb..5aad7d6eb3c16ce79811d6a66773a0087e69cc1c 100644
--- a/lib/python/dbrepo/RestClient.py
+++ b/lib/python/dbrepo/RestClient.py
@@ -5,7 +5,9 @@ import logging
import requests
from pydantic import TypeAdapter
from tusclient.client import TusClient
+from pandas import DataFrame
+from dbrepo.UploadClient import UploadClient
from dbrepo.api.dto import *
from dbrepo.api.exceptions import ResponseCodeError, UsernameExistsError, EmailExistsError, NotExistsError, \
ForbiddenError, MalformedError, NameExistsError, QueryStoreError, MetadataConsistencyError, ExternalSystemError, \
@@ -15,8 +17,8 @@ from dbrepo.api.exceptions import ResponseCodeError, UsernameExistsError, EmailE
class RestClient:
"""
The RestClient class for communicating with the DBRepo REST API. All parameters can be set also via environment \
- variables, e.g. set endpoint with DBREPO_ENDPOINT, username with DBREPO_USERNAME, etc. You can override the \
- constructor parameters with the environment variables.
+ variables, e.g. set endpoint with REST_API_ENDPOINT, username with REST_API_USERNAME, etc. You can override \
+ the constructor parameters with the environment variables.
:param endpoint: The REST API endpoint. Optional. Default: "http://gateway-service"
:param username: The REST API username. Optional.
@@ -38,11 +40,11 @@ class RestClient:
logging.getLogger('urllib3').setLevel(logging.INFO)
logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-6s %(message)s', level=logging.DEBUG,
stream=sys.stdout)
- self.endpoint = os.environ.get('DBREPO_ENDPOINT', endpoint)
- self.username = os.environ.get('DBREPO_USERNAME', username)
- self.password = os.environ.get('DBREPO_PASSWORD', password)
- if os.environ.get('DBREPO_SECURE') is not None:
- self.secure = os.environ.get('DBREPO_SECURE') == 'True'
+ self.endpoint = os.environ.get('REST_API_ENDPOINT', endpoint)
+ self.username = os.environ.get('REST_API_USERNAME', username)
+ self.password = os.environ.get('REST_API_PASSWORD', password)
+ if os.environ.get('REST_API_SECURE') is not None:
+ self.secure = os.environ.get('REST_API_SECURE') == 'True'
else:
self.secure = secure
@@ -86,9 +88,11 @@ class RestClient:
raise UploadError(f'Failed to upload the file to {self.endpoint}')
return filename
- def whoami(self) -> str:
+ def whoami(self) -> str | None:
"""
Print the username.
+
+ :returns: The username, if set.
"""
if self.username is not None:
logging.info(f"{self.username}")
@@ -124,7 +128,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return User.parse_raw(body)
+ return User.model_validate(body)
if response.status_code == 404:
raise NotExistsError(f'Failed to find user with id {user_id}')
raise ResponseCodeError(
@@ -151,7 +155,7 @@ class RestClient:
payload=CreateUser(username=username, password=password, email=email))
if response.status_code == 201:
body = response.json()
- return UserBrief.parse_raw(body)
+ return UserBrief.model_validate(body)
if response.status_code == 403:
raise ForbiddenError(f'Failed to update user password: not allowed')
if response.status_code == 404:
@@ -186,7 +190,7 @@ class RestClient:
orcid=orcid))
if response.status_code == 202:
body = response.json()
- return User.parse_raw(body)
+ return User.model_validate(body)
if response.status_code == 400:
raise ResponseCodeError(f'Failed to update user: invalid values')
if response.status_code == 403:
@@ -215,7 +219,7 @@ class RestClient:
response = self._wrapper(method="put", url=url, force_auth=True, payload=UpdateUserTheme(theme=theme))
if response.status_code == 202:
body = response.json()
- return User.parse_raw(body)
+ return User.model_validate(body)
if response.status_code == 400:
raise ResponseCodeError(f'Failed to update user theme: invalid values')
if response.status_code == 403:
@@ -244,7 +248,7 @@ class RestClient:
response = self._wrapper(method="put", url=url, force_auth=True, payload=UpdateUserPassword(password=password))
if response.status_code == 202:
body = response.json()
- return User.parse_raw(body)
+ return User.model_validate(body)
if response.status_code == 400:
raise ResponseCodeError(f'Failed to update user password: invalid values')
if response.status_code == 403:
@@ -286,7 +290,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return Container.parse_raw(body)
+ return Container.model_validate(body)
if response.status_code == 404:
raise NotExistsError(f'Failed to get container: not found')
raise ResponseCodeError(f'Failed to get container: response code: {response.status_code} is not 200 (OK)')
@@ -333,7 +337,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return Database.parse_raw(body)
+ return Database.model_validate(body)
if response.status_code == 404:
raise NotExistsError(f'Failed to find database with id {database_id}')
raise ResponseCodeError(
@@ -359,7 +363,7 @@ class RestClient:
payload=CreateDatabase(name=name, container_id=container_id, is_public=is_public))
if response.status_code == 201:
body = response.json()
- return Database.parse_raw(body)
+ return Database.model_validate(body)
if response.status_code == 403:
raise ForbiddenError(f'Failed to create database: not allowed')
if response.status_code == 404:
@@ -385,7 +389,7 @@ class RestClient:
response = self._wrapper(method="put", url=url, force_auth=True, payload=ModifyVisibility(is_public=is_public))
if response.status_code == 202:
body = response.json()
- return Database.parse_raw(body)
+ return Database.model_validate(body)
if response.status_code == 403:
raise ForbiddenError(f'Failed to update database visibility: not allowed')
if response.status_code == 404:
@@ -409,7 +413,7 @@ class RestClient:
response = self._wrapper(method="put", url=url, force_auth=True, payload=ModifyOwner(id=user_id))
if response.status_code == 202:
body = response.json()
- return Database.parse_raw(body)
+ return Database.model_validate(body)
if response.status_code == 403:
raise ForbiddenError(f'Failed to update database visibility: not allowed')
if response.status_code == 404:
@@ -442,7 +446,7 @@ class RestClient:
columns=columns, constraints=constraints))
if response.status_code == 201:
body = response.json()
- return Table.parse_raw(body)
+ return Table.model_validate(body)
if response.status_code == 400:
raise MalformedError(f'Failed to create table: service rejected malformed payload')
if response.status_code == 403:
@@ -488,7 +492,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return Table.parse_raw(body)
+ return Table.model_validate(body)
if response.status_code == 403:
raise ForbiddenError(f'Failed to find table: not allowed')
if response.status_code == 404:
@@ -556,7 +560,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return View.parse_raw(body)
+ return View.model_validate(body)
if response.status_code == 403:
raise ForbiddenError(f'Failed to find view: not allowed')
if response.status_code == 404:
@@ -584,7 +588,7 @@ class RestClient:
payload=CreateView(name=name, query=query, is_public=is_public))
if response.status_code == 201:
body = response.json()
- return View.parse_raw(body)
+ return View.model_validate(body)
if response.status_code == 400 or response.status_code == 423:
raise MalformedError(f'Failed to create view: service rejected malformed payload')
if response.status_code == 403 or response.status_code == 405:
@@ -616,7 +620,8 @@ class RestClient:
raise NotExistsError(f'Failed to delete view: not found')
raise ResponseCodeError(f'Failed to delete view: response code: {response.status_code} is not 202 (ACCEPTED)')
- def get_view_data(self, database_id: int, view_id: int, page: int = 0, size: int = 10) -> Result:
+ def get_view_data(self, database_id: int, view_id: int, page: int = 0, size: int = 10,
+ df: bool = False) -> Result | DataFrame:
"""
Get data of a view in a database with given database id and view id.
@@ -624,6 +629,7 @@ class RestClient:
:param view_id: The view id.
:param page: The result pagination number. Optional. Default: 0.
:param size: The result pagination size. Optional. Default: 10.
+ :param df: If true, the result is returned as Pandas DataFrame. Optional. Default: False.
:returns: The result of the view query, if successful.
@@ -639,7 +645,10 @@ class RestClient:
response = self._wrapper(method="get", url=url, params=params)
if response.status_code == 200:
body = response.json()
- return Result.parse_raw(body)
+ res = Result.model_validate(body)
+ if df:
+ return DataFrame.from_records(res.result)
+ return res
if response.status_code == 400:
raise MalformedError(f'Failed to get view data: service rejected malformed payload')
if response.status_code == 403:
@@ -649,7 +658,7 @@ class RestClient:
raise ResponseCodeError(f'Failed to get view data: response code: {response.status_code} is not 200 (OK)')
def get_table_data(self, database_id: int, table_id: int, page: int = 0, size: int = 10,
- timestamp: datetime.datetime = None) -> Result:
+ timestamp: datetime.datetime = None, df: bool = False) -> Result | DataFrame:
"""
Get data of a table in a database with given database id and table id.
@@ -658,6 +667,7 @@ class RestClient:
:param page: The result pagination number. Optional. Default: 0.
:param size: The result pagination size. Optional. Default: 10.
:param timestamp: The query execution time. Optional.
+ :param df: If true, the result is returned as Pandas DataFrame. Optional. Default: False.
:returns: The result of the view query, if successful.
@@ -676,7 +686,10 @@ class RestClient:
response = self._wrapper(method="get", url=url, params=params)
if response.status_code == 200:
body = response.json()
- return Result.parse_raw(body)
+ res = Result.model_validate(body)
+ if df:
+ return DataFrame.from_records(res.result)
+ return res
if response.status_code == 400:
raise MalformedError(f'Failed to get table data: service rejected malformed payload')
if response.status_code == 403:
@@ -736,8 +749,8 @@ class RestClient:
:raises NotExistsError: If the table does not exist.
:raises MalformedError: If the payload is rejected by the service (e.g. LOB data could not be imported).
"""
- upload = UploadClient(endpoint=self.endpoint)
- filename = upload.upload(file_path=file_path)
+ client = UploadClient(endpoint=self.endpoint)
+ filename = client.upload(file_path=file_path)
url = f'/api/database/{database_id}/table/{table_id}/data/import'
response = self._wrapper(method="post", url=url, force_auth=True,
payload=Import(location=filename, separator=separator, quote=quote,
@@ -790,7 +803,7 @@ class RestClient:
response = self._wrapper(method="get", url=url, params=params)
if response.status_code == 202:
body = response.json()
- return DatatypeAnalysis.parse_raw(body)
+ return DatatypeAnalysis.model_validate(body)
if response.status_code == 400 or response.status_code == 500:
raise MalformedError(f'Failed to analyse data types: service rejected malformed payload')
if response.status_code == 404:
@@ -826,7 +839,7 @@ class RestClient:
response = self._wrapper(method="get", url=url, params=params)
if response.status_code == 202:
body = response.json()
- return KeyAnalysis.parse_raw(body)
+ return KeyAnalysis.model_validate(body)
if response.status_code == 400 or response.status_code == 500:
raise MalformedError(f'Failed to analyse data types: service rejected malformed payload')
if response.status_code == 404:
@@ -851,7 +864,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 202:
body = response.json()
- return TableStatistics.parse_raw(body)
+ return TableStatistics.model_validate(body)
if response.status_code == 400:
raise MalformedError(f'Failed to analyse table statistics: service rejected malformed payload')
if response.status_code == 404:
@@ -994,7 +1007,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return DatabaseAccess.parse_raw(body).type
+ return DatabaseAccess.model_validate(body).type
if response.status_code == 403:
raise ForbiddenError(f'Failed to get database access: not allowed')
if response.status_code == 404:
@@ -1020,7 +1033,7 @@ class RestClient:
response = self._wrapper(method="post", url=url, force_auth=True, payload=CreateAccess(type=type))
if response.status_code == 202:
body = response.json()
- return DatabaseAccess.parse_raw(body).type
+ return DatabaseAccess.model_validate(body).type
if response.status_code == 400:
raise MalformedError(f'Failed to create database access: service rejected malformed payload')
if response.status_code == 403 or response.status_code == 405:
@@ -1049,7 +1062,7 @@ class RestClient:
response = self._wrapper(method="put", url=url, force_auth=True, payload=UpdateAccess(type=type))
if response.status_code == 202:
body = response.json()
- return DatabaseAccess.parse_raw(body).type
+ return DatabaseAccess.model_validate(body).type
if response.status_code == 400:
raise MalformedError(f'Failed to update database access: service rejected malformed payload')
if response.status_code == 403 or response.status_code == 405:
@@ -1113,7 +1126,7 @@ class RestClient:
payload=ExecuteQuery(statement=query, timestamp=timestamp))
if response.status_code == 202:
body = response.json()
- return Result.parse_raw(body)
+ return Result.model_validate(body)
if response.status_code == 400:
raise MalformedError(f'Failed to execute query: service rejected malformed payload')
if response.status_code == 403:
@@ -1128,7 +1141,7 @@ class RestClient:
f'Failed to execute query: response code: {response.status_code} is not 202 (ACCEPTED)')
def get_query_data(self, database_id: int, query_id: int, page: int = 0, size: int = 10,
- file_path: str = None) -> Result:
+ df: bool = False) -> Result | DataFrame:
"""
Re-executes a query in a database with given database id and query id.
@@ -1137,7 +1150,7 @@ class RestClient:
:param page: The result pagination number. Optional. Default: 0.
:param size: The result pagination size. Optional. Default: 10.
:param size: The result pagination size. Optional. Default: 10.
- :param file_path: The file path where the result should be saved. Optional.
+ :param df: If true, the result is returned as Pandas DataFrame. Optional. Default: False.
:returns: The result set, if successful.
@@ -1148,22 +1161,17 @@ class RestClient:
:raises QueryStoreError: The query store rejected the query.
:raises MetadataConsistencyError: The service failed to parse columns from the metadata database.
"""
- stream = False
headers = {}
url = f'/api/database/{database_id}/query/{query_id}/data'
if page is not None and size is not None:
url += f'?page={page}&size={size}'
- if file_path is not None:
- stream = True
- headers = {'Accept': 'text/csv'}
- response = self._wrapper(method="get", url=url, headers=headers, stream=stream)
+ response = self._wrapper(method="get", url=url, headers=headers)
if response.status_code == 200:
- if file_path is None:
- body = response.json()
- return Result.parse_raw(body)
- else:
- with open(file_path, "w") as f:
- f.write(response.content.decode("utf-8"))
+ body = response.json()
+ res = Result.model_validate(body)
+ if df:
+ return DataFrame.from_records(res.result)
+ return res
if response.status_code == 400:
raise MalformedError(f'Failed to re-execute query: service rejected malformed payload')
if response.status_code == 403 or response.status_code == 405:
@@ -1233,7 +1241,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return Query.parse_raw(body)
+ return Query.model_validate(body)
if response.status_code == 404:
raise NotExistsError(f'Failed to find query: not found')
if response.status_code == 403 or response.status_code == 405:
@@ -1295,7 +1303,7 @@ class RestClient:
response = self._wrapper(method="put", url=url, force_auth=True, payload=UpdateQuery(persist=persist))
if response.status_code == 202:
body = response.json()
- return Query.parse_raw(body)
+ return Query.model_validate(body)
if response.status_code == 403 or response.status_code == 405:
raise ForbiddenError(f'Failed to update query: not allowed')
if response.status_code == 404:
@@ -1349,7 +1357,7 @@ class RestClient:
response = self._wrapper(method="post", url=url, force_auth=True, payload=payload)
if response.status_code == 201:
body = response.json()
- return Identifier.parse_raw(body)
+ return Identifier.model_validate(body)
if response.status_code == 400:
raise MalformedError(f'Failed to create identifier: service rejected malformed payload')
if response.status_code == 403 or response.status_code == 405:
@@ -1376,7 +1384,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return Identifier.parse_raw(body)
+ return Identifier.model_validate(body)
if response.status_code == 404:
raise NotExistsError(f'Failed to suggest identifier: not found or not supported')
raise ResponseCodeError(f'Failed to suggest identifier: response code: {response.status_code} is not 200 (OK)')
@@ -1442,7 +1450,7 @@ class RestClient:
payload=UpdateColumn(concept_uri=concept_uri, unit_uri=unit_uri))
if response.status_code == 202:
body = response.json()
- return Column.parse_raw(body)
+ return Column.model_validate(body)
if response.status_code == 400:
raise MalformedError(f'Failed to update column: service rejected malformed payload')
if response.status_code == 403:
diff --git a/lib/python/dbrepo/UploadClient.py b/lib/python/dbrepo/UploadClient.py
new file mode 100644
index 0000000000000000000000000000000000000000..236453cb700cb6a1634c9910b66686543fda0aab
--- /dev/null
+++ b/lib/python/dbrepo/UploadClient.py
@@ -0,0 +1,39 @@
+import logging
+import os
+import re
+import sys
+from tusclient import client
+
+
+class UploadClient:
+ """
+ The UploadClient class for communicating with the DBRepo REST API. All parameters can be set also via environment \
+ variables, e.g. set endpoint with DBREPO_ENDPOINT, username with DBREPO_USERNAME, etc. You can override the \
+ constructor parameters with the environment variables.
+
+ :param endpoint: The REST API endpoint. Optional. Default: "http://gateway-service/api/upload/files"
+ """
+ endpoint: str = None
+
+ def __init__(self, endpoint: str = 'http://gateway-service/api/upload/files') -> None:
+ logging.getLogger('requests').setLevel(logging.INFO)
+ logging.getLogger('urllib3').setLevel(logging.INFO)
+ logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-6s %(message)s', level=logging.DEBUG,
+ stream=sys.stdout)
+ self.endpoint = os.environ.get('REST_UPLOAD_ENDPOINT', endpoint)
+
+ def upload(self, file_path: str) -> str:
+ """
+ Imports a file through the Upload Service into the Storage Service.
+
+ :param file_path: The file path on the local machine.
+
+ :returns: Filename on the Storage Service, if successful.
+ """
+ tus_client = client.TusClient(url=self.endpoint)
+ uploader = tus_client.uploader(file_path=file_path)
+ uploader.upload()
+ m = re.search('\\/([a-f0-9]+)\\+', uploader.url)
+ filename = m.group(0)[1:-1]
+ logging.debug(f'uploaded file {file_path} to storage service with key: {filename}')
+ return filename
diff --git a/lib/python/debug.py b/lib/python/debug.py
deleted file mode 100644
index afcb58609017fba7c51b6465e2a637f55a97037e..0000000000000000000000000000000000000000
--- a/lib/python/debug.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from dbrepo.RestClient import RestClient
-
-client = RestClient(endpoint="https://dbrepo1.ec.tuwien.ac.at", username="foo",
- password="bar")
-client.get_licenses()
diff --git a/lib/python/docs/guide/amqp-client.rst b/lib/python/docs/guide/amqp-client.rst
index 0bb970292c2391305c45d3feff0b9791a92f05af..7f87621f43f486dc7692a20920b9ba36894a8338 100644
--- a/lib/python/docs/guide/amqp-client.rst
+++ b/lib/python/docs/guide/amqp-client.rst
@@ -1,8 +1,5 @@
-AMQP Client
-===========
-
-.. warning::
- This documentation is a work in progress.
+AMQP API Client
+===============
.. automodule:: dbrepo.AmqpClient
:members:
diff --git a/lib/python/docs/guide/rest-client.rst b/lib/python/docs/guide/rest-client.rst
index 4af546168d3a4142566967982339dbba4f8f0032..09ed4019436d79c019e6a9f22803d92cc72ed419 100644
--- a/lib/python/docs/guide/rest-client.rst
+++ b/lib/python/docs/guide/rest-client.rst
@@ -1,8 +1,5 @@
-REST Client
-===========
-
-.. warning::
- This documentation is a work in progress.
+REST API Client
+===============
.. automodule:: dbrepo.RestClient
:members:
diff --git a/lib/python/docs/guide/upload-client.rst b/lib/python/docs/guide/upload-client.rst
new file mode 100644
index 0000000000000000000000000000000000000000..4070b0908894914d0391e0ebbffa37d81c499a88
--- /dev/null
+++ b/lib/python/docs/guide/upload-client.rst
@@ -0,0 +1,6 @@
+Upload API Client
+=================
+
+.. automodule:: dbrepo.UploadClient
+ :members:
+ :no-index:
diff --git a/lib/python/docs/index.rst b/lib/python/docs/index.rst
index e084d4c3b2c07f1b601165b7d1d7f518bf9c8a2d..d1916a2e2aafc6173b2003379a08ae7140b98559 100644
--- a/lib/python/docs/index.rst
+++ b/lib/python/docs/index.rst
@@ -1,18 +1,28 @@
-DBRepo Python Library
-=====================
+DBRepo Python Library documentation
+===================================
-.. image:: https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/badges/release.svg
- :alt: DBRepo latest release version
- :target: https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services
+Use the DBRepo SDK to create, update, configure and manage DBRepo services such as the Data Service to get data as
+Pandas `DataFrame <https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html>`_ for analysis. The DBRepo SDK
+provides an object-oriented API as well as low-level access to DBRepo services.
-.. image:: https://img.shields.io/pypi/dm/dbrepo
- :alt: PyPI downloads per month
- :target: https://pypi.org/project/dbrepo/__APPVERSION__/
+.. note::
+ The SDK has been implemented and documented for DBRepo version 1.4.2, earlier versions are not supported.
-.. warning::
- This documentation is a work in progress.
+Quickstart
+----------
-REST Client
+Find numerous quickstart examples on
+the `DBRepo website <https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/usage-overview/>`_.
+
+AMQP API Client
+-----------
+
+.. toctree::
+ :maxdepth: 2
+
+ guide/amqp-client
+
+REST API Client
-----------
.. toctree::
@@ -20,13 +30,13 @@ REST Client
guide/rest-client
-AMQP Client
+Upload API Client
-----------
.. toctree::
:maxdepth: 2
- guide/amqp-client
+ guide/upload-client
Indices and tables
==================
diff --git a/lib/python/package.sh b/lib/python/package.sh
new file mode 100755
index 0000000000000000000000000000000000000000..4478dbf962d7dba47fc787061b5761d2b53a68eb
--- /dev/null
+++ b/lib/python/package.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+PRE_RELEASE=""
+if [ "${CI_COMMIT_BRANCH:8:8}" = "master" ]; then
+ PRE_RELEASE="rc${CI_PIPELINE_ID}"
+fi
+sed -i -e "s/__APPVERSION__/${APP_VERSION}${PRE_RELEASE}/g" ./lib/python/pyproject.toml ./lib/python/setup.py ./lib/python/README.md
+python -m build --sdist ./lib/python
+python -m build --wheel ./lib/python
diff --git a/lib/python/pyproject.toml b/lib/python/pyproject.toml
index 6dfae10925cfb7d47f9d0272d5ab59ddb884bcc1..43baf9a5f11d5ca0f9806173dd148816168bd9d5 100644
--- a/lib/python/pyproject.toml
+++ b/lib/python/pyproject.toml
@@ -23,7 +23,8 @@ dependencies = [
"requests >= 2.31",
"pika",
"pydantic",
- "tuspy"
+ "tuspy",
+ "pandas"
]
[build-system]
diff --git a/lib/python/release.sh b/lib/python/release.sh
new file mode 100755
index 0000000000000000000000000000000000000000..5e2b32683188c63d1c7fe36a80bbdfce6728ff45
--- /dev/null
+++ b/lib/python/release.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+cat ${CI_PIPYRC} | base64 -d > .pypirc
+python -m twine upload --config-file .pypirc --verbose --repository pypi ./lib/python/dist/dbrepo-*
diff --git a/lib/python/tests/test_analyse.py b/lib/python/tests/test_analyse.py
index c87cb22018ba81634b9e3526e7051688ad31ee97..a4668fedc500dbd8c6ebebf9edb6f3885156f5eb 100644
--- a/lib/python/tests/test_analyse.py
+++ b/lib/python/tests/test_analyse.py
@@ -13,7 +13,7 @@ class AnalyseTest(unittest.TestCase):
with requests_mock.Mocker() as mock:
exp = KeyAnalysis(keys={'id': 0, 'firstname': 1, 'lastname': 2})
# mock
- mock.get('/api/analyse/keys', json=exp.model_dump_json(), status_code=202)
+ mock.get('/api/analyse/keys', json=exp.model_dump(), status_code=202)
# test
response = RestClient().analyse_keys(file_path='f705a7bd0cb2d5e37ab2b425036810a2', separator=',',
upload=False)
diff --git a/lib/python/tests/test_container.py b/lib/python/tests/test_container.py
index d948e337db607a0898f1cfb641a32d10bf821878..e9988f19ca9dd1567d48295c7c4e3184acc30b83 100644
--- a/lib/python/tests/test_container.py
+++ b/lib/python/tests/test_container.py
@@ -94,7 +94,7 @@ class ContainerTest(unittest.TestCase):
]),
hash="f829dd8a884182d0da846f365dee1221fd16610a14c81b8f9f295ff162749e50")
# mock
- mock.get('/api/container/1', json=exp.model_dump_json())
+ mock.get('/api/container/1', json=exp.model_dump())
# test
response = RestClient().get_container(container_id=1)
self.assertEqual(exp, response)
diff --git a/lib/python/tests/test_database.py b/lib/python/tests/test_database.py
index 9df3a11d5a3a26795ff98bab60429a12f37ad333..017a17445ab4a7e4eb0e56002f1a444da0208118 100644
--- a/lib/python/tests/test_database.py
+++ b/lib/python/tests/test_database.py
@@ -110,7 +110,7 @@ class DatabaseTest(unittest.TestCase):
)
with requests_mock.Mocker() as mock:
# mock
- mock.get('/api/database/1', json=exp.model_dump_json())
+ mock.get('/api/database/1', json=exp.model_dump())
# test
response = RestClient().get_database(1)
self.assertEqual(exp, response)
@@ -178,7 +178,7 @@ class DatabaseTest(unittest.TestCase):
)
with requests_mock.Mocker() as mock:
# mock
- mock.post('/api/database', json=exp.model_dump_json(), status_code=201)
+ mock.post('/api/database', json=exp.model_dump(), status_code=201)
# test
client = RestClient(username="a", password="b")
response = client.create_database(name='test', container_id=1, is_public=True)
@@ -253,7 +253,7 @@ class DatabaseTest(unittest.TestCase):
)
with requests_mock.Mocker() as mock:
# mock
- mock.put('/api/database/1', json=exp.model_dump_json(), status_code=202)
+ mock.put('/api/database/1', json=exp.model_dump(), status_code=202)
# test
client = RestClient(username="a", password="b")
response = client.update_database_visibility(database_id=1, is_public=True)
@@ -328,7 +328,7 @@ class DatabaseTest(unittest.TestCase):
)
with requests_mock.Mocker() as mock:
# mock
- mock.put('/api/database/1/owner', json=exp.model_dump_json(), status_code=202)
+ mock.put('/api/database/1/owner', json=exp.model_dump(), status_code=202)
# test
client = RestClient(username="a", password="b")
response = client.update_database_owner(database_id=1, user_id='abdbf897-e599-4e5a-a3f0-7529884ea011')
@@ -376,7 +376,7 @@ class DatabaseTest(unittest.TestCase):
attributes=UserAttributes(theme='light')))
with requests_mock.Mocker() as mock:
# mock
- mock.get('/api/database/1/access', json=exp.model_dump_json())
+ mock.get('/api/database/1/access', json=exp.model_dump())
# test
response = RestClient().get_database_access(database_id=1)
self.assertEqual(response, AccessType.READ)
@@ -408,7 +408,7 @@ class DatabaseTest(unittest.TestCase):
attributes=UserAttributes(theme='light')))
with requests_mock.Mocker() as mock:
# mock
- mock.post('/api/database/1/access/abdbf897-e599-4e5a-a3f0-7529884ea011', json=exp.model_dump_json(),
+ mock.post('/api/database/1/access/abdbf897-e599-4e5a-a3f0-7529884ea011', json=exp.model_dump(),
status_code=202)
# test
client = RestClient(username="a", password="b")
@@ -470,7 +470,7 @@ class DatabaseTest(unittest.TestCase):
attributes=UserAttributes(theme='light')))
with requests_mock.Mocker() as mock:
# mock
- mock.put('/api/database/1/access/abdbf897-e599-4e5a-a3f0-7529884ea011', json=exp.model_dump_json(),
+ mock.put('/api/database/1/access/abdbf897-e599-4e5a-a3f0-7529884ea011', json=exp.model_dump(),
status_code=202)
# test
client = RestClient(username="a", password="b")
diff --git a/lib/python/tests/test_identifier.py b/lib/python/tests/test_identifier.py
index c39e7a99a5a54747da17274a1d89acec3ba9c2eb..64ebfe1f5166dcc1de0f6fb5c3fb28283ce8bf53 100644
--- a/lib/python/tests/test_identifier.py
+++ b/lib/python/tests/test_identifier.py
@@ -34,7 +34,7 @@ class IdentifierTest(unittest.TestCase):
type=RelatedIdentifierType.DOI)],
creators=[IdentifierCreator(id=5, creator_name='Carberry, Josiah')])
# mock
- mock.post('/api/identifier', json=exp.model_dump_json(), status_code=201)
+ mock.post('/api/identifier', json=exp.model_dump(), status_code=201)
# test
client = RestClient(username="a", password="b")
response = client.create_identifier(database_id=1, type=IdentifierType.VIEW,
@@ -137,7 +137,7 @@ class IdentifierTest(unittest.TestCase):
creators=[IdentifierCreator(id=5, creator_name='Carberry, Josiah',
name_identifier='https://orcid.org/0000-0002-1825-0097')])
# mock
- mock.get('/api/identifier?url=https://orcid.org/0000-0002-1825-0097', json=exp.model_dump_json())
+ mock.get('/api/identifier?url=https://orcid.org/0000-0002-1825-0097', json=exp.model_dump())
# test
response = RestClient().suggest_identifier("https://orcid.org/0000-0002-1825-0097")
self.assertEqual(exp, response)
diff --git a/lib/python/tests/test_query.py b/lib/python/tests/test_query.py
index 76bd95d8d34e7531806191fb6214c6f45d526612..0d75b8afc5d34e18f432d38e6796541a4522c038 100644
--- a/lib/python/tests/test_query.py
+++ b/lib/python/tests/test_query.py
@@ -1,9 +1,12 @@
import unittest
+from json import dumps
+from typing import Any
import requests_mock
import datetime
from dbrepo.RestClient import RestClient
+from pandas import DataFrame
from dbrepo.api.dto import Result, Query, User, UserAttributes, QueryType
from dbrepo.api.exceptions import MalformedError, NotExistsError, ForbiddenError, QueryStoreError, \
@@ -18,7 +21,7 @@ class QueryTest(unittest.TestCase):
headers=[{'id': 0, 'username': 1}],
id=None)
# mock
- mock.post('/api/database/1/query', json=exp.model_dump_json(), status_code=202)
+ mock.post('/api/database/1/query', json=exp.model_dump(), status_code=202)
# test
client = RestClient(username="a", password="b")
response = client.execute_query(database_id=1, page=0, size=10,
@@ -114,7 +117,7 @@ class QueryTest(unittest.TestCase):
result_number=None,
identifiers=[])
# mock
- mock.get('/api/database/1/query/6', json=exp.model_dump_json())
+ mock.get('/api/database/1/query/6', json=exp.model_dump())
# test
response = RestClient().get_query(database_id=1, query_id=6)
self.assertEqual(exp, response)
@@ -237,11 +240,24 @@ class QueryTest(unittest.TestCase):
headers=[{'id': 0, 'username': 1}],
id=6)
# mock
- mock.get('/api/database/1/query/6/data', json=exp.model_dump_json())
+ mock.get('/api/database/1/query/6/data', json=exp.model_dump())
# test
response = RestClient().get_query_data(database_id=1, query_id=6)
self.assertEqual(exp, response)
+ def test_get_query_data_dataframe_succeeds(self):
+ with requests_mock.Mocker() as mock:
+ res = Result(result=[{'id': 1, 'username': 'foo'}, {'id': 2, 'username': 'bar'}],
+ headers=[{'id': 0, 'username': 1}],
+ id=6)
+ exp = DataFrame.from_records(res.model_dump()['result'])
+ # mock
+ mock.get('/api/database/1/query/6/data', json=res.model_dump())
+ # test
+ response = RestClient().get_query_data(database_id=1, query_id=6, df=True)
+ self.assertEqual(exp.shape, response.shape)
+ self.assertTrue(DataFrame.equals(exp, response))
+
def test_get_query_data_not_allowed_fails(self):
with requests_mock.Mocker() as mock:
# mock
diff --git a/lib/python/tests/test_table.py b/lib/python/tests/test_table.py
index 68f947386e4441a8aa290bf224b55cf15e26496c..286d908c917b857f5c939d1b83fbd968ead89890 100644
--- a/lib/python/tests/test_table.py
+++ b/lib/python/tests/test_table.py
@@ -1,9 +1,11 @@
import unittest
+from json import dumps
import requests_mock
import datetime
from dbrepo.RestClient import RestClient
+from pandas import DataFrame
from dbrepo.api.dto import Table, CreateTableConstraints, UserAttributes, User, Column, Constraints, ColumnType, Result, \
Concept, Unit, TableStatistics, ColumnStatistic
@@ -42,7 +44,7 @@ class TableTest(unittest.TestCase):
is_null_allowed=False)])
with requests_mock.Mocker() as mock:
# mock
- mock.post('/api/database/1/table', json=exp.model_dump_json(), status_code=201)
+ mock.post('/api/database/1/table', json=exp.model_dump(), status_code=201)
# test
client = RestClient(username="a", password="b")
response = client.create_table(database_id=1, name="Test", description="Test Table", columns=[],
@@ -179,7 +181,7 @@ class TableTest(unittest.TestCase):
is_public=True,
is_null_allowed=False)])
# mock
- mock.get('/api/database/1/table/2', json=exp.model_dump_json())
+ mock.get('/api/database/1/table/2', json=exp.model_dump())
# test
response = RestClient().get_table(database_id=1, table_id=2)
self.assertEqual(exp, response)
@@ -250,11 +252,24 @@ class TableTest(unittest.TestCase):
headers=[{'id': 0, 'username': 1}],
id=None)
# mock
- mock.get('/api/database/1/table/9/data', json=exp.model_dump_json())
+ mock.get('/api/database/1/table/9/data', json=exp.model_dump())
# test
response = RestClient().get_table_data(database_id=1, table_id=9)
self.assertEqual(exp, response)
+ def test_get_table_data_dataframe_succeeds(self):
+ with requests_mock.Mocker() as mock:
+ res = Result(result=[{'id': 1, 'username': 'foo'}, {'id': 2, 'username': 'bar'}],
+ headers=[{'id': 0, 'username': 1}],
+ id=None)
+ exp = DataFrame.from_records(res.model_dump()['result'])
+ # mock
+ mock.get('/api/database/1/table/9/data', json=res.model_dump())
+ # test
+ response = RestClient().get_table_data(database_id=1, table_id=9, df=True)
+ self.assertEqual(exp.shape, response.shape)
+ self.assertTrue(DataFrame.equals(exp, response))
+
def test_get_table_data_malformed_fails(self):
with requests_mock.Mocker() as mock:
# mock
@@ -558,7 +573,7 @@ class TableTest(unittest.TestCase):
name="liters per square meter"),
is_null_allowed=False)
# mock
- mock.put('/api/database/1/table/2/column/1', json=exp.model_dump_json(), status_code=202)
+ mock.put('/api/database/1/table/2/column/1', json=exp.model_dump(), status_code=202)
# test
client = RestClient(username="a", password="b")
response = client.update_table_column(database_id=1, table_id=2, column_id=1,
@@ -622,7 +637,7 @@ class TableTest(unittest.TestCase):
exp = TableStatistics(
columns={"id": ColumnStatistic(val_min=1.0, val_max=9.0, mean=5.0, median=5.0, std_dev=2.73)})
# mock
- mock.get('/api/analyse/database/1/table/2/statistics', json=exp.model_dump_json(), status_code=202)
+ mock.get('/api/analyse/database/1/table/2/statistics', json=exp.model_dump(), status_code=202)
# test
response = RestClient().analyse_table_statistics(database_id=1, table_id=2)
self.assertEqual(exp, response)
diff --git a/lib/python/tests/test_user.py b/lib/python/tests/test_user.py
index d5afaf2d01c90517dee6ecf3fffeec96347d466b..67698a8fd58b811f58a89a7ba9d1c5245f2f242c 100644
--- a/lib/python/tests/test_user.py
+++ b/lib/python/tests/test_user.py
@@ -53,7 +53,7 @@ class UserTest(unittest.TestCase):
with requests_mock.Mocker() as mock:
exp = UserBrief(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise')
# mock
- mock.post('http://gateway-service/api/user', json=exp.model_dump_json(), status_code=201)
+ mock.post('http://gateway-service/api/user', json=exp.model_dump(), status_code=201)
# test
response = RestClient().create_user(username='mweise', password='s3cr3t', email='mweise@example.com')
self.assertEqual(exp, response)
@@ -62,7 +62,7 @@ class UserTest(unittest.TestCase):
with requests_mock.Mocker() as mock:
exp = UserBrief(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise')
# mock
- mock.post('http://gateway-service/api/user', json=exp.model_dump_json(), status_code=400)
+ mock.post('http://gateway-service/api/user', json=exp.model_dump(), status_code=400)
# test
try:
response = RestClient().create_user(username='mweise', password='s3cr3t', email='mweise@example.com')
@@ -73,7 +73,7 @@ class UserTest(unittest.TestCase):
with requests_mock.Mocker() as mock:
exp = UserBrief(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise')
# mock
- mock.post('http://gateway-service/api/user', json=exp.model_dump_json(), status_code=403)
+ mock.post('http://gateway-service/api/user', json=exp.model_dump(), status_code=403)
# test
try:
response = RestClient().create_user(username='mweise', password='s3cr3t', email='mweise@example.com')
@@ -84,7 +84,7 @@ class UserTest(unittest.TestCase):
with requests_mock.Mocker() as mock:
exp = UserBrief(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise')
# mock
- mock.post('http://gateway-service/api/user', json=exp.model_dump_json(), status_code=409)
+ mock.post('http://gateway-service/api/user', json=exp.model_dump(), status_code=409)
# test
try:
response = RestClient().create_user(username='mweise', password='s3cr3t', email='mweise@example.com')
@@ -95,7 +95,7 @@ class UserTest(unittest.TestCase):
with requests_mock.Mocker() as mock:
exp = UserBrief(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise')
# mock
- mock.post('http://gateway-service/api/user', json=exp.model_dump_json(), status_code=404)
+ mock.post('http://gateway-service/api/user', json=exp.model_dump(), status_code=404)
# test
try:
response = RestClient().create_user(username='mweise', password='s3cr3t', email='mweise@example.com')
@@ -106,7 +106,7 @@ class UserTest(unittest.TestCase):
with requests_mock.Mocker() as mock:
exp = UserBrief(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise')
# mock
- mock.post('http://gateway-service/api/user', json=exp.model_dump_json(), status_code=417)
+ mock.post('http://gateway-service/api/user', json=exp.model_dump(), status_code=417)
# test
try:
response = RestClient().create_user(username='mweise', password='s3cr3t', email='mweise@example.com')
@@ -119,7 +119,7 @@ class UserTest(unittest.TestCase):
attributes=UserAttributes(theme='dark'))
# mock
mock.get('http://gateway-service/api/user/8638c043-5145-4be8-a3e4-4b79991b0a16',
- json=exp.model_dump_json())
+ json=exp.model_dump())
# test
response = RestClient().get_user(user_id='8638c043-5145-4be8-a3e4-4b79991b0a16')
self.assertEqual(exp, response)
@@ -140,7 +140,7 @@ class UserTest(unittest.TestCase):
attributes=UserAttributes(theme='dark'))
# mock
mock.put('http://gateway-service/api/user/8638c043-5145-4be8-a3e4-4b79991b0a16', status_code=202,
- json=exp.model_dump_json())
+ json=exp.model_dump())
# test
client = RestClient(username="a", password="b")
response = client.update_user(user_id='8638c043-5145-4be8-a3e4-4b79991b0a16', firstname='Martin')
@@ -195,7 +195,7 @@ class UserTest(unittest.TestCase):
attributes=UserAttributes(theme='dark'))
# mock
mock.put('http://gateway-service/api/user/8638c043-5145-4be8-a3e4-4b79991b0a16/theme', status_code=202,
- json=exp.model_dump_json())
+ json=exp.model_dump())
# test
client = RestClient(username="a", password="b")
response = client.update_user_theme(user_id='8638c043-5145-4be8-a3e4-4b79991b0a16', theme='dark')
@@ -250,7 +250,7 @@ class UserTest(unittest.TestCase):
attributes=UserAttributes(theme='dark'))
# mock
mock.put('http://gateway-service/api/user/8638c043-5145-4be8-a3e4-4b79991b0a16/password', status_code=202,
- json=exp.model_dump_json())
+ json=exp.model_dump())
# test
client = RestClient(username="a", password="b")
response = client.update_user_password(user_id='8638c043-5145-4be8-a3e4-4b79991b0a16',
diff --git a/lib/python/tests/test_view.py b/lib/python/tests/test_view.py
index 16eb6f4c518aa79c8d1e9ed0d9d7aed1c599614b..2a61cab89ee2a4a0fa1778c598da27e00ea9910b 100644
--- a/lib/python/tests/test_view.py
+++ b/lib/python/tests/test_view.py
@@ -1,9 +1,11 @@
import unittest
+from json import dumps
import requests_mock
import datetime
from dbrepo.RestClient import RestClient
+from pandas import DataFrame
from dbrepo.api.dto import UserAttributes, User, View, Result
from dbrepo.api.exceptions import ForbiddenError, NotExistsError, MalformedError, AuthenticationError
@@ -76,7 +78,7 @@ class ViewTest(unittest.TestCase):
last_modified=datetime.datetime(2024, 1, 1, 0, 0, 0, 0, datetime.timezone.utc),
identifiers=[])
# mock
- mock.get('/api/database/1/view/3', json=exp.model_dump_json())
+ mock.get('/api/database/1/view/3', json=exp.model_dump())
# test
response = RestClient().get_view(database_id=1, view_id=3)
self.assertEqual(exp, response)
@@ -117,7 +119,7 @@ class ViewTest(unittest.TestCase):
last_modified=datetime.datetime(2024, 1, 1, 0, 0, 0, 0, datetime.timezone.utc),
identifiers=[])
# mock
- mock.post('/api/database/1/view', json=exp.model_dump_json(), status_code=201)
+ mock.post('/api/database/1/view', json=exp.model_dump(), status_code=201)
# test
client = RestClient(username="a", password="b")
response = client.create_view(database_id=1, name="Data", is_public=True,
@@ -217,11 +219,24 @@ class ViewTest(unittest.TestCase):
headers=[{'id': 0, 'username': 1}],
id=None)
# mock
- mock.get('/api/database/1/view/3/data', json=exp.model_dump_json())
+ mock.get('/api/database/1/view/3/data', json=exp.model_dump())
# test
response = RestClient().get_view_data(database_id=1, view_id=3)
self.assertEqual(exp, response)
+ def test_get_view_data_dataframe_succeeds(self):
+ with requests_mock.Mocker() as mock:
+ res = Result(result=[{'id': 1, 'username': 'foo'}, {'id': 2, 'username': 'bar'}],
+ headers=[{'id': 0, 'username': 1}],
+ id=None)
+ exp = DataFrame.from_records(res.model_dump()['result'])
+ # mock
+ mock.get('/api/database/1/view/3/data', json=res.model_dump())
+ # test
+ response: DataFrame = RestClient().get_view_data(database_id=1, view_id=3, df=True)
+ self.assertEqual(exp.shape, response.shape)
+ self.assertTrue(DataFrame.equals(exp, response))
+
def test_get_view_data_malformed_fails(self):
with requests_mock.Mocker() as mock:
# mock