From 0fff43ed25de51d0f9d67d99094bb39443a66412 Mon Sep 17 00:00:00 2001
From: Martin Weise <martin.weise@tuwien.ac.at>
Date: Wed, 10 Apr 2024 11:47:06 +0200
Subject: [PATCH] Added documentation
---
lib/python/Makefile | 26 +++++++++
lib/python/README.md | 74 +++++++++++++++----------
lib/python/dbrepo/AmqpClient.py | 16 +++---
lib/python/dbrepo/RestClient.py | 23 ++++----
lib/python/dbrepo/UploadClient.py | 39 +++++++++++++
lib/python/docs/guide/amqp-client.rst | 7 +--
lib/python/docs/guide/rest-client.rst | 7 +--
lib/python/docs/guide/upload-client.rst | 6 ++
lib/python/docs/index.rst | 36 +++++++-----
9 files changed, 166 insertions(+), 68 deletions(-)
create mode 100644 lib/python/Makefile
create mode 100644 lib/python/dbrepo/UploadClient.py
create mode 100644 lib/python/docs/guide/upload-client.rst
diff --git a/lib/python/Makefile b/lib/python/Makefile
new file mode 100644
index 0000000000..4b9e18e3ad
--- /dev/null
+++ b/lib/python/Makefile
@@ -0,0 +1,26 @@
+all:
+
+clean:
+ rm -rf ./python/dist/* ./docs/build/* ./dist/*
+
+install:
+ pipenv install
+
+docs: clean
+ sphinx-apidoc -o ./docs/source ./dbrepo
+ sphinx-build -M html ./docs/ ./docs/build/
+
+check:
+ python3 ./python/setup.py develop
+
+build: clean
+ python3 -m build --sdist .
+ python3 -m build --wheel .
+
+deploy: build
+ python3 -m twine upload --config-file ~/.pypirc --verbose --repository pypi ./dist/dbrepo-*
+
+deploy-test: build
+ python3 -m twine upload --config-file ~/.pypirc --verbose --repository testpypi ./dist/dbrepo-*
+
+FORCE: ;
\ No newline at end of file
diff --git a/lib/python/README.md b/lib/python/README.md
index 0cb742a451..610c6eded4 100644
--- a/lib/python/README.md
+++ b/lib/python/README.md
@@ -32,33 +32,33 @@ print(f"Analysis result: {analysis}")
# create table
table = client.create_table(database_id=1,
- name="Sensor Data",
- constraints=CreateTableConstraints(
- checks=['precipitation >= 0'],
- uniques=[['precipitation']]),
- columns=[CreateTableColumn(name="date",
- type=ColumnType.DATE,
- dfid=3, # YYYY-MM-dd
- primary_key=True,
- null_allowed=False),
- CreateTableColumn(name="precipitation",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True),
- CreateTableColumn(name="lat",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True),
- CreateTableColumn(name="lng",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True)])
+ name="Sensor Data",
+ constraints=CreateTableConstraints(
+ checks=['precipitation >= 0'],
+ uniques=[['precipitation']]),
+ columns=[CreateTableColumn(name="date",
+ type=ColumnType.DATE,
+ dfid=3, # YYYY-MM-dd
+ primary_key=True,
+ null_allowed=False),
+ CreateTableColumn(name="precipitation",
+ type=ColumnType.DECIMAL,
+ size=10,
+ d=4,
+ primary_key=False,
+ null_allowed=True),
+ CreateTableColumn(name="lat",
+ type=ColumnType.DECIMAL,
+ size=10,
+ d=4,
+ primary_key=False,
+ null_allowed=True),
+ CreateTableColumn(name="lng",
+ type=ColumnType.DECIMAL,
+ size=10,
+ d=4,
+ primary_key=False,
+ null_allowed=True)])
print(f"Create table result {table}")
# -> (id=1, internal_name=sensor_data, ...)
@@ -83,7 +83,25 @@ print(f"Finished.")
queries ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//usage-overview/#export-subset))
- Get data from tables/views/subsets
-## Future
+## Configure
+
+All credentials can optionally be set/overridden with environment variables. This is especially useful when sharing
+Jupyter Notebooks by creating an invisible `.env` file and loading it:
+
+```
+REST_API_ENDPOINT="https://test.dbrepo.tuwien.ac.at"
+REST_API_USERNAME="foo"
+REST_API_PASSWORD="bar"
+REST_API_SECURE="True"
+AMQP_API_HOST="https://test.dbrepo.tuwien.ac.at"
+AMQP_API_PORT="5672"
+AMQP_API_USERNAME="foo"
+AMQP_API_PASSWORD="bar"
+AMQP_API_VIRTUAL_HOST="/"
+REST_UPLOAD_ENDPOINT="https://test.dbrepo.tuwien.ac.at/api/upload/files"
+```
+
+## Roadmap
- Searching
diff --git a/lib/python/dbrepo/AmqpClient.py b/lib/python/dbrepo/AmqpClient.py
index c9fcdc05ff..29f7e261ec 100644
--- a/lib/python/dbrepo/AmqpClient.py
+++ b/lib/python/dbrepo/AmqpClient.py
@@ -2,8 +2,8 @@ import dataclasses
import os
import pika
import sys
-import logging
import json
+import logging
from dbrepo.api.dto import CreateData
@@ -32,16 +32,18 @@ class AmqpClient:
broker_virtual_host: str = '/',
username: str = None,
password: str = None) -> None:
+ logging.getLogger('requests').setLevel(logging.INFO)
+ logging.getLogger('urllib3').setLevel(logging.INFO)
logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-6s %(message)s', level=logging.DEBUG,
stream=sys.stdout)
- self.broker_host = os.environ.get('DBREPO_BROKER_HOST', broker_host)
- self.broker_port = os.environ.get('DBREPO_BROKER_PORT', broker_port)
- if os.environ.get('DBREPO_BROKER_VIRTUAL_HOST') is not None:
- self.broker_virtual_host = os.environ.get('DBREPO_BROKER_VIRTUAL_HOST')
+ self.broker_host = os.environ.get('AMQP_API_HOST', broker_host)
+ self.broker_port = os.environ.get('AMQP_API_PORT', broker_port)
+ if os.environ.get('AMQP_API_VIRTUAL_HOST') is not None:
+ self.broker_virtual_host = os.environ.get('AMQP_API_VIRTUAL_HOST')
else:
self.broker_virtual_host = broker_virtual_host
- self.username = os.environ.get('DBREPO_USERNAME', username)
- self.password = os.environ.get('DBREPO_PASSWORD', password)
+ self.username = os.environ.get('AMQP_API_USERNAME', username)
+ self.password = os.environ.get('AMQP_API_PASSWORD', password)
def publish(self, exchange: str, routing_key: str, data=dict) -> None:
"""
diff --git a/lib/python/dbrepo/RestClient.py b/lib/python/dbrepo/RestClient.py
index 619fec9a66..5aad7d6eb3 100644
--- a/lib/python/dbrepo/RestClient.py
+++ b/lib/python/dbrepo/RestClient.py
@@ -7,6 +7,7 @@ from pydantic import TypeAdapter
from tusclient.client import TusClient
from pandas import DataFrame
+from dbrepo.UploadClient import UploadClient
from dbrepo.api.dto import *
from dbrepo.api.exceptions import ResponseCodeError, UsernameExistsError, EmailExistsError, NotExistsError, \
ForbiddenError, MalformedError, NameExistsError, QueryStoreError, MetadataConsistencyError, ExternalSystemError, \
@@ -16,8 +17,8 @@ from dbrepo.api.exceptions import ResponseCodeError, UsernameExistsError, EmailE
class RestClient:
"""
The RestClient class for communicating with the DBRepo REST API. All parameters can be set also via environment \
- variables, e.g. set endpoint with DBREPO_ENDPOINT, username with DBREPO_USERNAME, etc. You can override the \
- constructor parameters with the environment variables.
+ variables, e.g. set endpoint with REST_API_ENDPOINT, username with REST_API_USERNAME, etc. You can override \
+ the constructor parameters with the environment variables.
:param endpoint: The REST API endpoint. Optional. Default: "http://gateway-service"
:param username: The REST API username. Optional.
@@ -39,11 +40,11 @@ class RestClient:
logging.getLogger('urllib3').setLevel(logging.INFO)
logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-6s %(message)s', level=logging.DEBUG,
stream=sys.stdout)
- self.endpoint = os.environ.get('DBREPO_ENDPOINT', endpoint)
- self.username = os.environ.get('DBREPO_USERNAME', username)
- self.password = os.environ.get('DBREPO_PASSWORD', password)
- if os.environ.get('DBREPO_SECURE') is not None:
- self.secure = os.environ.get('DBREPO_SECURE') == 'True'
+ self.endpoint = os.environ.get('REST_API_ENDPOINT', endpoint)
+ self.username = os.environ.get('REST_API_USERNAME', username)
+ self.password = os.environ.get('REST_API_PASSWORD', password)
+ if os.environ.get('REST_API_SECURE') is not None:
+ self.secure = os.environ.get('REST_API_SECURE') == 'True'
else:
self.secure = secure
@@ -87,9 +88,11 @@ class RestClient:
raise UploadError(f'Failed to upload the file to {self.endpoint}')
return filename
- def whoami(self) -> str:
+ def whoami(self) -> str | None:
"""
Print the username.
+
+ :returns: The username, if set.
"""
if self.username is not None:
logging.info(f"{self.username}")
@@ -746,8 +749,8 @@ class RestClient:
:raises NotExistsError: If the table does not exist.
:raises MalformedError: If the payload is rejected by the service (e.g. LOB data could not be imported).
"""
- upload = UploadClient(endpoint=self.endpoint)
- filename = upload.upload(file_path=file_path)
+ client = UploadClient(endpoint=self.endpoint)
+ filename = client.upload(file_path=file_path)
url = f'/api/database/{database_id}/table/{table_id}/data/import'
response = self._wrapper(method="post", url=url, force_auth=True,
payload=Import(location=filename, separator=separator, quote=quote,
diff --git a/lib/python/dbrepo/UploadClient.py b/lib/python/dbrepo/UploadClient.py
new file mode 100644
index 0000000000..236453cb70
--- /dev/null
+++ b/lib/python/dbrepo/UploadClient.py
@@ -0,0 +1,39 @@
+import logging
+import os
+import re
+import sys
+from tusclient import client
+
+
+class UploadClient:
+ """
+ The UploadClient class for communicating with the DBRepo REST API. All parameters can be set also via environment \
+ variables, e.g. set endpoint with DBREPO_ENDPOINT, username with DBREPO_USERNAME, etc. You can override the \
+ constructor parameters with the environment variables.
+
+ :param endpoint: The REST API endpoint. Optional. Default: "http://gateway-service/api/upload/files"
+ """
+ endpoint: str = None
+
+ def __init__(self, endpoint: str = 'http://gateway-service/api/upload/files') -> None:
+ logging.getLogger('requests').setLevel(logging.INFO)
+ logging.getLogger('urllib3').setLevel(logging.INFO)
+ logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-6s %(message)s', level=logging.DEBUG,
+ stream=sys.stdout)
+ self.endpoint = os.environ.get('REST_UPLOAD_ENDPOINT', endpoint)
+
+ def upload(self, file_path: str) -> str:
+ """
+ Imports a file through the Upload Service into the Storage Service.
+
+ :param file_path: The file path on the local machine.
+
+ :returns: Filename on the Storage Service, if successful.
+ """
+ tus_client = client.TusClient(url=self.endpoint)
+ uploader = tus_client.uploader(file_path=file_path)
+ uploader.upload()
+ m = re.search('\\/([a-f0-9]+)\\+', uploader.url)
+ filename = m.group(0)[1:-1]
+ logging.debug(f'uploaded file {file_path} to storage service with key: {filename}')
+ return filename
diff --git a/lib/python/docs/guide/amqp-client.rst b/lib/python/docs/guide/amqp-client.rst
index 0bb970292c..7f87621f43 100644
--- a/lib/python/docs/guide/amqp-client.rst
+++ b/lib/python/docs/guide/amqp-client.rst
@@ -1,8 +1,5 @@
-AMQP Client
-===========
-
-.. warning::
- This documentation is a work in progress.
+AMQP API Client
+===============
.. automodule:: dbrepo.AmqpClient
:members:
diff --git a/lib/python/docs/guide/rest-client.rst b/lib/python/docs/guide/rest-client.rst
index 4af546168d..09ed401943 100644
--- a/lib/python/docs/guide/rest-client.rst
+++ b/lib/python/docs/guide/rest-client.rst
@@ -1,8 +1,5 @@
-REST Client
-===========
-
-.. warning::
- This documentation is a work in progress.
+REST API Client
+===============
.. automodule:: dbrepo.RestClient
:members:
diff --git a/lib/python/docs/guide/upload-client.rst b/lib/python/docs/guide/upload-client.rst
new file mode 100644
index 0000000000..4070b09088
--- /dev/null
+++ b/lib/python/docs/guide/upload-client.rst
@@ -0,0 +1,6 @@
+Upload API Client
+=================
+
+.. automodule:: dbrepo.UploadClient
+ :members:
+ :no-index:
diff --git a/lib/python/docs/index.rst b/lib/python/docs/index.rst
index e084d4c3b2..d1916a2e2a 100644
--- a/lib/python/docs/index.rst
+++ b/lib/python/docs/index.rst
@@ -1,18 +1,28 @@
-DBRepo Python Library
-=====================
+DBRepo Python Library documentation
+===================================
-.. image:: https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/badges/release.svg
- :alt: DBRepo latest release version
- :target: https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services
+Use the DBRepo SDK to create, update, configure and manage DBRepo services such as the Data Service to get data as
+Pandas `DataFrame <https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html>`_ for analysis. The DBRepo SDK
+provides an object-oriented API as well as low-level access to DBRepo services.
-.. image:: https://img.shields.io/pypi/dm/dbrepo
- :alt: PyPI downloads per month
- :target: https://pypi.org/project/dbrepo/__APPVERSION__/
+.. note::
+ The SDK has been implemented and documented for DBRepo version 1.4.2, earlier versions are not supported.
-.. warning::
- This documentation is a work in progress.
+Quickstart
+----------
-REST Client
+Find numerous quickstart examples on
+the `DBRepo website <https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/usage-overview/>`_.
+
+AMQP API Client
+-----------
+
+.. toctree::
+ :maxdepth: 2
+
+ guide/amqp-client
+
+REST API Client
-----------
.. toctree::
@@ -20,13 +30,13 @@ REST Client
guide/rest-client
-AMQP Client
+Upload API Client
-----------
.. toctree::
:maxdepth: 2
- guide/amqp-client
+ guide/upload-client
Indices and tables
==================
--
GitLab