diff --git a/.docs/api/python.md b/.docs/api/python.md
index ab6b2b69a6b6ca9606232eca534686bc6c0b71ab..3f1a8928542dd01b16e549482215f35fecf2a3ea 100644
--- a/.docs/api/python.md
+++ b/.docs/api/python.md
@@ -33,89 +33,78 @@ This package supports Python 3.11+.
## Quickstart
-Create a table and import a .csv file from your computer.
+Get public data from a table as pandas `DataFrame`:
```python
from dbrepo.RestClient import RestClient
-from dbrepo.api.dto import CreateTableColumn, ColumnType, CreateTableConstraints
-client = RestClient(endpoint='https://test.dbrepo.tuwien.ac.at', username="foo",
- password="bar")
-
-# analyse csv
-analysis = client.analyse_datatypes(file_path="sensor.csv", separator=",")
-print(f"Analysis result: {analysis}")
-# -> columns=(date=date, precipitation=decimal, lat=decimal, lng=decimal), separator=,
-# line_termination=\n
-
-# create table
-table = client.create_table(database_id=1,
- name="Sensor Data",
- constraints=CreateTableConstraints(checks=['precipitation >= 0'],
- uniques=[['precipitation']]),
- columns=[CreateTableColumn(name="date",
- type=ColumnType.DATE,
- dfid=3, # YYYY-MM-dd
- primary_key=True,
- null_allowed=False),
- CreateTableColumn(name="precipitation",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True),
- CreateTableColumn(name="lat",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True),
- CreateTableColumn(name="lng",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True)])
-print(f"Create table result {table}")
-# -> (id=1, internal_name=sensor_data, ...)
-
-client.import_table_data(database_id=1, table_id=1, file_path="sensor.csv", separator=",",
- skip_lines=1, line_encoding="\n")
-print(f"Finished.")
+client = RestClient(endpoint="https://dbrepo1.ec.tuwien.ac.at")
+# Get a small data slice of just three rows
+df = client.get_table_data(database_id=7, table_id=13, page=0, size=3, df=True)
+print(df)
+# x_coord component unit ... value stationid meantype
+# 0 16.52617 Feinstaub (PM10) µg/m³ ... 21.0 01:0001 HMW
+# 1 16.52617 Feinstaub (PM10) µg/m³ ... 23.0 01:0001 HMW
+# 2 16.52617 Feinstaub (PM10) µg/m³ ... 26.0 01:0001 HMW
+#
+# [3 rows x 12 columns]
```
-The library is well-documented, please see the [full documentation](../sphinx) or
-the [PyPI page](https://pypi.org/project/dbrepo/).
+Import data into a table:
-## Supported Features & Best-Practices
+```python
+import pandas as pd
+from dbrepo.RestClient import RestClient
-- Manage user account ([docs](../usage-overview/#create-user-account))
-- Manage databases ([docs](../usage-overview/#create-database))
-- Manage database access & visibility ([docs](../usage-overview/#private-database-access))
-- Import dataset ([docs](../usage-overview/#private-database-access))
-- Create persistent identifiers ([docs](../usage-overview/#assign-database-pid))
-- Execute queries ([docs](../usage-overview/#export-subset))
-- Get data from tables/views/subsets
+client = RestClient(endpoint="https://dbrepo1.ec.tuwien.ac.at", username="foo",
+ password="bar")
+df = pd.DataFrame(data={'x_coord': 16.52617, 'component': 'Feinstaub (PM10)',
+ 'unit': 'µg/m³', ...})
+client.import_table_data(database_id=7, table_id=13, file_name_or_data_frame=df)
+```
-## Secrets
+## Supported Features & Best-Practices
-It is not recommended to store credentials directly in the notebook as they will be versioned with git, etc. Use
-environment variables instead:
+- Manage user
+ account ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#create-user-account))
+- Manage
+ databases ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//usage-overview/#create-database))
+- Manage database access &
+ visibility ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#create-database))
+- Import
+ dataset ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#import-dataset))
+- Create persistent
+ identifiers ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#assign-database-pid))
+- Execute
+ queries ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#export-subset))
+- Get data from tables/views/subsets
-```properties title=".env"
-DBREPO_ENDPOINT=https://test.dbrepo.tuwien.ac.at
-DBREPO_USERNAME=foo
-DBREPO_PASSWORD=bar
-DBREPO_SECURE=True
+## Configure
+
+All credentials can optionally be set/overridden with environment variables. This is especially useful when sharing
+Jupyter Notebooks by creating an invisible `.env` file and loading it:
+
+``` title=".env"
+REST_API_ENDPOINT="https://dbrepo1.ec.tuwien.ac.at"
+REST_API_USERNAME="foo"
+REST_API_PASSWORD="bar"
+REST_API_SECURE="True"
+AMQP_API_HOST="https://dbrepo1.ec.tuwien.ac.at"
+AMQP_API_PORT="5672"
+AMQP_API_USERNAME="foo"
+AMQP_API_PASSWORD="bar"
+AMQP_API_VIRTUAL_HOST="dbrepo"
+REST_UPLOAD_ENDPOINT="https://dbrepo1.ec.tuwien.ac.at/api/upload/files"
```
-Then use the default constructor of the `RestClient` to e.g. analyse a CSV. Your secrets are automatically passed:
+You can disable logging by setting the log level to e.g. `INFO`:
-```python title="analysis.py"
+```python
from dbrepo.RestClient import RestClient
-
-client = RestClient()
-analysis = client.analyse_datatypes(file_path="sensor.csv", separator=",")
+import logging
+logging.getLogger().setLevel(logging.INFO)
+...
+client = RestClient(...)
```
## Future
diff --git a/.docs/examples/air.md b/.docs/examples/air.md
new file mode 100644
index 0000000000000000000000000000000000000000..1e4d9ddeddf83d5861ad421ba8d00f7397508f46
--- /dev/null
+++ b/.docs/examples/air.md
@@ -0,0 +1,38 @@
+---
+author: Martin Weise
+---
+
+## tl;dr
+
+[:fontawesome-solid-database: Dataset](https://handle.stage.datacite.org/10.82556/gd17-aq82){ .md-button .md-button--primary target="_blank" }
+[:material-file-document: Archive](https://doi.org/10.48436/mtha8-w2406){ .md-button .md-button--secondary target="_blank" }
+
+## Description
+
+This digital record contains historical air pollution and air quality data from approximately 20 air monitoring stations
+in Vienna, spanning the years from 1980 to 2021. The data was provided by the Umweltbundesamt and is stored in its
+original form in this record. This record forms the basis of an analysis carried out in a bachelor's thesis at the TU
+Wien.
+
+## Solution
+
+
+<figure markdown>
+{ .img-border }
+<figcaption>Figure 1: Jupyter Notebook accessing data on DBRepo using the Python Library.</figcaption>
+</figure>
+
+## DBRepo Features
+
+- [x] Import complex dataset
+- [x] System versioning
+- [x] Subset exploration
+- [x] Aggregated views
+- [x] Precise & PID of queries tables
+- [x] External data access for analysis
+
+## Acknowledgement
+
+This work was part of a cooperation with the [Umweltbundesamt](https://www.umweltbundesamt.at/).
+
+<img src="../../images/logos/umweltbundesamt.png" width=100 />
\ No newline at end of file
diff --git a/.docs/examples/covid-19.md b/.docs/examples/covid-19.md
new file mode 100644
index 0000000000000000000000000000000000000000..ba986e76e9275de13e0be646e908ad8e38256d65
--- /dev/null
+++ b/.docs/examples/covid-19.md
@@ -0,0 +1,25 @@
+---
+author: Martin Weise
+---
+
+## tl;dr
+
+[:fontawesome-solid-database: Dataset](https://dbrepo1.ec.tuwien.ac.at/pid/15){ .md-button .md-button--primary target="_blank" }
+[:simple-github: Archive](https://github.com/CSSEGISandData/COVID-19){ .md-button .md-button--secondary target="_blank" }
+
+## Description
+
+This dataset contains the daily COVID-19 data provided publicly
+by [Johns Hopkins University](https://coronavirus.jhu.edu/about/how-to-use-our-data).
+
+## Solution
+
+We imported their daily snapshots provided as 1145 versioned .csv files from their Git repository archive and imported
+them daily into DBRepo as system-versioned data that can be queried. During the time of this project the COVID-19
+pandemic was still ongoing and therefore daily snapshots demanded a correct import script to be maintained.
+
+## DBRepo Features
+
+- [x] Data pipeline from Git repository
+- [x] System versioning
+- [x] Subset exploration
diff --git a/.docs/examples/influenza.md b/.docs/examples/influenza.md
deleted file mode 100644
index 074c413d31f3f5e1d590d2491c6562677ee75a39..0000000000000000000000000000000000000000
--- a/.docs/examples/influenza.md
+++ /dev/null
@@ -1,21 +0,0 @@
----
-author: Martin Weise
----
-
-## tl;dr
-
-tbd
-
-## Description
-
-TBD
-
-## Solution
-
-TBD
-
-## DBRepo Features
-
-- [x] Import through CSV-dataset upload
-- [x] Data views implementing embargo period (24 hours)
-- [x] External access from Grafana Dashboard
diff --git a/.docs/examples/music.md b/.docs/examples/music.md
new file mode 100644
index 0000000000000000000000000000000000000000..02a848d59d02142a20a1ffeb403ca634eeb7fc59
--- /dev/null
+++ b/.docs/examples/music.md
@@ -0,0 +1,34 @@
+---
+author: Martin Weise
+---
+
+## tl;dr
+
+[:fontawesome-solid-database: Dataset](https://dbrepo1.ec.tuwien.ac.at/pid/34){ .md-button .md-button--primary target="_blank" }
+[:material-file-document: Archive](https://gitlab.tuwien.ac.at/martin.weise/fairnb){ .md-button .md-button--secondary target="_blank" }
+
+## Description
+
+We use a dataset collected by [Aljanaki et al.](https://www2.projects.science.uu.nl/memotion/emotifydata/), consisting
+of 400 MP3 music files, each having a playtime of one minute and labeled with one of four genres: rock, pop, classical
+and electronic, each genre contains 100 files, the genre will be used as label for the ML model. Then by generating MFCC
+vectors and training a SVM, the ML-model can classify emotions of the provided .mp3 files with and accuracy of 76.25%.
+
+<figure markdown>
+{ .img-border }
+<figcaption>Figure 1: Accuracy of predictions matrix in Jupyter Notebook.</figcaption>
+</figure>
+
+## Solution
+
+DBRepo is used as relational data storage of the raw- and aggregated features, prediction results and the splits of the
+training- and test data. For each of the 400 .mp3 files, 40 MFCC feature vectors are generated. This data is stored
+in aggregated form in the [`aggregated_features`](https://dbrepo1.ec.tuwien.ac.at/pid/47) table.
+
+## DBRepo Features
+
+- [x] Database as storage for machine learning data
+- [x] System versioning
+- [x] Subset exploration
+- [x] Precise & PID of database tables
+- [x] External data access for analysis
diff --git a/.docs/examples/power.md b/.docs/examples/power.md
index 4e86f6c30a2a313e34b6b9616c5968db62b30ea5..e85b1b98ce32b675829a67866fb3bef4340cd965 100644
--- a/.docs/examples/power.md
+++ b/.docs/examples/power.md
@@ -15,7 +15,7 @@ maintenance, quality of products and ultimately process efficiency and -producti
<figure markdown>
-<figcaption>Figure 1: aaaa from <a href="https://publik.tuwien.ac.at/files/PubDat_252294.pdf">Hacksteiner (2016)</a>.</figcaption>
+<figcaption>Figure 1: Total power usage of machine floor TU Pilotfabrik, image from <a href="https://publik.tuwien.ac.at/files/PubDat_252294.pdf">Hacksteiner (2016)</a>.</figcaption>
</figure>
## Solution
diff --git a/.docs/examples/survey.md b/.docs/examples/survey.md
new file mode 100644
index 0000000000000000000000000000000000000000..88eea632e066e4ce3ed250d47e7ba6e5c46b5c8f
--- /dev/null
+++ b/.docs/examples/survey.md
@@ -0,0 +1,38 @@
+---
+author: Martin Weise
+---
+
+## tl;dr
+
+[:fontawesome-solid-database: Dataset](https://handle.stage.datacite.org/10.82556/g2ac-vh88){ .md-button .md-button--primary target="_blank" }
+[:simple-jupyter: Notebook](https://binder.science.datalab.tuwien.ac.at/v2/git/https%3A%2F%2Fgitlab.tuwien.ac.at%2Fmartin.weise%2Ftres/HEAD){ .md-button .md-button--secondary target="_blank" }
+
+## Description
+
+This digital record contains historical air pollution and air quality data from approximately 20 air monitoring stations
+in Vienna, spanning the years from 1980 to 2021. The data was provided by the Umweltbundesamt and is stored in its
+original form in this record. This record forms the basis of an analysis carried out in a bachelor's thesis at the TU
+Wien.
+
+## Solution
+
+
+<figure markdown>
+{ .img-border }
+<figcaption>Figure 1: Jupyter Notebook accessing data on DBRepo using the Python Library.</figcaption>
+</figure>
+
+## DBRepo Features
+
+- [x] Import complex dataset
+- [x] System versioning
+- [x] Subset exploration
+- [x] Aggregated views
+- [x] Precise & PID of queries tables
+- [x] External data access for analysis
+
+## Acknowledgement
+
+This work was part of a cooperation with the [Umweltbundesamt](https://www.umweltbundesamt.at/).
+
+<img src="../../images/logos/umweltbundesamt.png" width=100 />
\ No newline at end of file
diff --git a/.docs/examples/xps-data.md b/.docs/examples/xps-data.md
new file mode 100644
index 0000000000000000000000000000000000000000..9b237c8c6c8ec2be919751475f810605a9ee1925
--- /dev/null
+++ b/.docs/examples/xps-data.md
@@ -0,0 +1,48 @@
+---
+author: Martin Weise
+---
+
+## tl;dr
+
+[:fontawesome-solid-database: Dataset](https://dbrepo1.ec.tuwien.ac.at/database/27/info){ .md-button .md-button--primary target="_blank" }
+[:simple-jupyter: Notebook](https://binder.science.datalab.tuwien.ac.at/v2/git/https%3A%2F%2Fgitlab.tuwien.ac.at%2Ffairdata%2Fxps/HEAD){ .md-button .md-button--secondary target="_blank" }
+
+## Description
+
+X-ray Photoelectron Spectroscopy (XPS) is one of the most used methods in material sciences. Irradiation of solid
+materials with X-ray radiation kicks out electrons from atoms that are near the atomic nucleus. With XPS data being
+highly reproducible once machine parameters are known and understood, the demand for creating a comprehensive database
+connecting material properties to compositions via XPS spectra becomes evident.
+
+## Solution
+
+We read XPS data from the VAMAS-encoded format and inserted it into a
+[database schema](https://gitlab.tuwien.ac.at/fairdata/xps/-/blob/e17860399b1b109c72b01888766f37193dde5870/sql/create_schema.sql)
+that captures the VAMAS-schema. It can then be read using the Python Library that executes a database query in SQL to
+obtain only the experiment data (c.f. [subset page](https://dbrepo1.ec.tuwien.ac.at/database/27/subset/10/info)).
+
+<figure markdown>
+{ .img-border }
+<figcaption>Figure 1: Jupyter Notebook accessing data on DBRepo using the Python Library.</figcaption>
+</figure>
+
+Using the DataFrame representation of the Python Library and the [`plotly`](https://pypi.org/project/plotly/) library,
+we can visualize the ordinate values directly in the Jupyter Notebook.
+
+<figure markdown>
+{ .img-border }
+<figcaption>Figure 2: Plot of ordinate values encoded within the experiment block.</figcaption>
+</figure>
+
+## DBRepo Features
+
+- [x] Data preservation of VAMAS-encoded XPS data
+- [x] Subset exploration
+- [x] External visualization of the database
+- [x] Replication of experiments using only open-source software
+
+## Acknowledgement
+
+This work was part of a cooperation with the [Institute of Applied Physics](http://www.iap.tuwien.ac.at/).
+
+<img src="../../images/logos/iap.png" width=100 />
\ No newline at end of file
diff --git a/.docs/images/logos/iap.png b/.docs/images/logos/iap.png
new file mode 100644
index 0000000000000000000000000000000000000000..763bd0580f2ffb49a283570717471d822ee7045c
Binary files /dev/null and b/.docs/images/logos/iap.png differ
diff --git a/.docs/images/logos/umweltbundesamt.png b/.docs/images/logos/umweltbundesamt.png
new file mode 100644
index 0000000000000000000000000000000000000000..4230cd264615bf50399d9742efaf49dfca445da6
Binary files /dev/null and b/.docs/images/logos/umweltbundesamt.png differ
diff --git a/.docs/images/screenshots/air-notebook.png b/.docs/images/screenshots/air-notebook.png
new file mode 100644
index 0000000000000000000000000000000000000000..8f1a21e405da9e2ac01d8b79475d40f8aa61924f
Binary files /dev/null and b/.docs/images/screenshots/air-notebook.png differ
diff --git a/.docs/images/screenshots/mfcc-jupyter.png b/.docs/images/screenshots/mfcc-jupyter.png
new file mode 100644
index 0000000000000000000000000000000000000000..36905661efce77254652ea60ea41629426695ba2
Binary files /dev/null and b/.docs/images/screenshots/mfcc-jupyter.png differ
diff --git a/.docs/images/screenshots/xps-chart.png b/.docs/images/screenshots/xps-chart.png
index 91768d7f7babe3c20a57fde415872ae268d45c73..c064c2f2feefeafe831b3d628f5f9bdd63ac6136 100644
Binary files a/.docs/images/screenshots/xps-chart.png and b/.docs/images/screenshots/xps-chart.png differ
diff --git a/.docs/images/screenshots/xps-notebook.png b/.docs/images/screenshots/xps-notebook.png
new file mode 100644
index 0000000000000000000000000000000000000000..a1031b751991eb41d66c1c291f921f13f7cd5304
Binary files /dev/null and b/.docs/images/screenshots/xps-notebook.png differ
diff --git a/.docs/why.md b/.docs/why.md
index 1e6e52cc7f2d0c932fd5640cdfde752d48862e30..2a269cdadb5032d5b482f1bda9b1e0c6804d1756 100644
--- a/.docs/why.md
+++ b/.docs/why.md
@@ -2,8 +2,6 @@
author: Martin Weise
---
-## Why use DBRepo?
-
Digital repositories see themselves more frequently encountered with the problem of making databases accessible in their
collection. Challenges revolve around organizing, searching and retrieving content stored within databases and
constitute a major technical burden as their internal representation greatly differs from static documents most digital
@@ -21,28 +19,29 @@ evolving, allows reproducing of query results and supports findable-, accessible
DBRepo makes your dataset searchable without extra effort: most metadata is generated automatically for data in your
databases. The fast and powerful OpenSearch database allows a fast retrieval of any information. Adding semantic mapping
-through a suggestion-feature, allows machines to properly understand the context of your data. [Learn more.](../system-services-search/)
+through a suggestion-feature, allows machines to properly understand the context of your data
+[Learn more](../concepts/search).
### Citable datasets
Adopting the recommendations of the RDA-WGDC, arbitrary subsets can be precisely, persistently identified using
system-versioned tables of MariaDB and the DataCite schema for minting DOIs. External systems i.e. metadata harvesters
-(OpenAIRE, Google Datasets) can access these datasets through OAI-PMH, JSON-LD and FAIR Signposting protocols.
-[Learn more.](../system-services-metadata/)
+(OpenAIRE, Google Datasets) can access these datasets through OAI-PMH, JSON-LD and FAIR Signposting protocols
+[Learn more](../concepts/pid).
### Powerful API for Data Scientists
With our strongly typed Python Library, Data Scientists can import, export and work with data from Jupyter Notebook or
Python script, optionally using Pandas DataFrames. For example: the AMQP API Client can collect continuous data from
-edge devices like sensors and store them asynchronous in DBRepo. [Learn more.](../usage-python/)
+edge devices like sensors and store them asynchronous in DBRepo [Learn more](../api/python).
### Cloud Native
Our lightweight Helm chart allows for installations on any cloud provider or private-cloud setting that has an
underlying PV storage provider. DBRepo can be installed from
the [Artifact Hub](https://artifacthub.io/packages/helm/dbrepo/dbrepo) repository. Databases are managed as MariaDB
-Galera Cluster with high degree of availability ensuring your data is always accessible.
-[Learn more.](../deployment-helm/)
+Galera Cluster with high degree of availability ensuring your data is always accessible
+[Learn more](../kubernetes).
## Demo Site
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c74b4cabacbf60544fa005acb720440040173fcf..c7c0cfca0b6f44025bc8e597ed2030a7a291d41f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -125,7 +125,7 @@ build-helm:
- echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
script:
- apk add sed helm curl
- - helm package ./helm/dbrepo --sign --key 'Martin Weise' --keyring ./secring.gpg --destination ./build
+ - helm package ./helm/dbrepo --destination ./build
verify-install-script:
image: docker.io/docker:24-dind
diff --git a/dbrepo-analyse-service/Pipfile.lock b/dbrepo-analyse-service/Pipfile.lock
index 9a4ce182e44f3a2d2decb81b875d0ba268ab27ba..fa863d3b7b63cffb1752ce9d5426f653a569b23f 100644
--- a/dbrepo-analyse-service/Pipfile.lock
+++ b/dbrepo-analyse-service/Pipfile.lock
@@ -167,19 +167,19 @@
},
"boto3": {
"hashes": [
- "sha256:7e8418b47dd43954a9088d504541bed8a42b6d06e712d02befba134c1c4d7c6d",
- "sha256:7f676daef674fe74f34ce4063228eccc6e60c811f574720e31f230296c4bf29a"
+ "sha256:b781d267dd5e7583966e05697f6bd45e2f46c01dc619ba0860b042963ee69296",
+ "sha256:c163fb7135a94e7b8c8c478a44071c843f05e212fa4bec3105f8a437ecbf1bcb"
],
"index": "pypi",
- "version": "==1.34.126"
+ "version": "==1.34.130"
},
"botocore": {
"hashes": [
- "sha256:7a8ccb6a7c02456757a984a3a44331b6f51c94cb8b9b287cd045122fd177a4b0",
- "sha256:7eff883c638fe30e0b036789df32d851e093d12544615a3b90062b42ac85bdbc"
+ "sha256:a242b3b0a836b14f308a309565cd63e88654cec238f9b73abbbd3c0526db4c81",
+ "sha256:a3b36e9dac1ed31c4cb3a5c5e540a7d8a9b90ff1d17f87734e674154b41776d8"
],
"markers": "python_version >= '3.8'",
- "version": "==1.34.126"
+ "version": "==1.34.130"
},
"certifi": {
"hashes": [
@@ -887,45 +887,54 @@
},
"numpy": {
"hashes": [
- "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b",
- "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818",
- "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20",
- "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0",
- "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010",
- "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a",
- "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea",
- "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c",
- "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71",
- "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110",
- "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be",
- "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a",
- "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a",
- "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5",
- "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed",
- "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd",
- "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c",
- "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e",
- "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0",
- "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c",
- "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a",
- "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b",
- "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0",
- "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6",
- "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2",
- "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a",
- "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30",
- "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218",
- "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5",
- "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07",
- "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2",
- "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4",
- "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764",
- "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef",
- "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3",
- "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"
+ "sha256:04494f6ec467ccb5369d1808570ae55f6ed9b5809d7f035059000a37b8d7e86f",
+ "sha256:0a43f0974d501842866cc83471bdb0116ba0dffdbaac33ec05e6afed5b615238",
+ "sha256:0e50842b2295ba8414c8c1d9d957083d5dfe9e16828b37de883f51fc53c4016f",
+ "sha256:0ec84b9ba0654f3b962802edc91424331f423dcf5d5f926676e0150789cb3d95",
+ "sha256:17067d097ed036636fa79f6a869ac26df7db1ba22039d962422506640314933a",
+ "sha256:1cde1753efe513705a0c6d28f5884e22bdc30438bf0085c5c486cdaff40cd67a",
+ "sha256:1e72728e7501a450288fc8e1f9ebc73d90cfd4671ebbd631f3e7857c39bd16f2",
+ "sha256:2635dbd200c2d6faf2ef9a0d04f0ecc6b13b3cad54f7c67c61155138835515d2",
+ "sha256:2ce46fd0b8a0c947ae047d222f7136fc4d55538741373107574271bc00e20e8f",
+ "sha256:34f003cb88b1ba38cb9a9a4a3161c1604973d7f9d5552c38bc2f04f829536609",
+ "sha256:354f373279768fa5a584bac997de6a6c9bc535c482592d7a813bb0c09be6c76f",
+ "sha256:38ecb5b0582cd125f67a629072fed6f83562d9dd04d7e03256c9829bdec027ad",
+ "sha256:3e8e01233d57639b2e30966c63d36fcea099d17c53bf424d77f088b0f4babd86",
+ "sha256:3f6bed7f840d44c08ebdb73b1825282b801799e325bcbdfa6bc5c370e5aecc65",
+ "sha256:4554eb96f0fd263041baf16cf0881b3f5dafae7a59b1049acb9540c4d57bc8cb",
+ "sha256:46e161722e0f619749d1cd892167039015b2c2817296104487cd03ed4a955995",
+ "sha256:49d9f7d256fbc804391a7f72d4a617302b1afac1112fac19b6c6cec63fe7fe8a",
+ "sha256:4d2f62e55a4cd9c58c1d9a1c9edaedcd857a73cb6fda875bf79093f9d9086f85",
+ "sha256:5f64641b42b2429f56ee08b4f427a4d2daf916ec59686061de751a55aafa22e4",
+ "sha256:63b92c512d9dbcc37f9d81b123dec99fdb318ba38c8059afc78086fe73820275",
+ "sha256:6d7696c615765091cc5093f76fd1fa069870304beaccfd58b5dcc69e55ef49c1",
+ "sha256:79e843d186c8fb1b102bef3e2bc35ef81160ffef3194646a7fdd6a73c6b97196",
+ "sha256:821eedb7165ead9eebdb569986968b541f9908979c2da8a4967ecac4439bae3d",
+ "sha256:84554fc53daa8f6abf8e8a66e076aff6ece62de68523d9f665f32d2fc50fd66e",
+ "sha256:8d83bb187fb647643bd56e1ae43f273c7f4dbcdf94550d7938cfc32566756514",
+ "sha256:903703372d46bce88b6920a0cd86c3ad82dae2dbef157b5fc01b70ea1cfc430f",
+ "sha256:9416a5c2e92ace094e9f0082c5fd473502c91651fb896bc17690d6fc475128d6",
+ "sha256:9a1712c015831da583b21c5bfe15e8684137097969c6d22e8316ba66b5baabe4",
+ "sha256:9c27f0946a3536403efb0e1c28def1ae6730a72cd0d5878db38824855e3afc44",
+ "sha256:a356364941fb0593bb899a1076b92dfa2029f6f5b8ba88a14fd0984aaf76d0df",
+ "sha256:a7039a136017eaa92c1848152827e1424701532ca8e8967fe480fe1569dae581",
+ "sha256:acd3a644e4807e73b4e1867b769fbf1ce8c5d80e7caaef0d90dcdc640dfc9787",
+ "sha256:ad0c86f3455fbd0de6c31a3056eb822fc939f81b1618f10ff3406971893b62a5",
+ "sha256:b4c76e3d4c56f145d41b7b6751255feefae92edbc9a61e1758a98204200f30fc",
+ "sha256:b6f6a8f45d0313db07d6d1d37bd0b112f887e1369758a5419c0370ba915b3871",
+ "sha256:c5a59996dc61835133b56a32ebe4ef3740ea5bc19b3983ac60cc32be5a665d54",
+ "sha256:c73aafd1afca80afecb22718f8700b40ac7cab927b8abab3c3e337d70e10e5a2",
+ "sha256:cee6cc0584f71adefe2c908856ccc98702baf95ff80092e4ca46061538a2ba98",
+ "sha256:cef04d068f5fb0518a77857953193b6bb94809a806bd0a14983a8f12ada060c9",
+ "sha256:cf5d1c9e6837f8af9f92b6bd3e86d513cdc11f60fd62185cc49ec7d1aba34864",
+ "sha256:e61155fae27570692ad1d327e81c6cf27d535a5d7ef97648a17d922224b216de",
+ "sha256:e7f387600d424f91576af20518334df3d97bc76a300a755f9a8d6e4f5cadd289",
+ "sha256:ed08d2703b5972ec736451b818c2eb9da80d66c3e84aed1deeb0c345fefe461b",
+ "sha256:fbd6acc766814ea6443628f4e6751d0da6593dae29c08c0b2606164db026970c",
+ "sha256:feff59f27338135776f6d4e2ec7aeeac5d5f7a08a83e80869121ef8164b74af9"
],
"index": "pypi",
- "version": "==1.26.4"
+ "version": "==2.0.0"
},
"opensearch-py": {
"hashes": [
@@ -1352,11 +1361,11 @@
},
"setuptools": {
"hashes": [
- "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4",
- "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0"
+ "sha256:01a1e793faa5bd89abc851fa15d0a0db26f160890c7102cd8dce643e886b47f5",
+ "sha256:d9b8b771455a97c8a9f3ab3448ebe0b29b5e105f1228bba41028be116985a267"
],
"markers": "python_version >= '3.8'",
- "version": "==70.0.0"
+ "version": "==70.1.0"
},
"six": {
"hashes": [
@@ -1400,11 +1409,11 @@
},
"urllib3": {
"hashes": [
- "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d",
- "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"
+ "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472",
+ "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"
],
"markers": "python_version >= '3.8'",
- "version": "==2.2.1"
+ "version": "==2.2.2"
},
"werkzeug": {
"hashes": [
@@ -1996,11 +2005,11 @@
},
"urllib3": {
"hashes": [
- "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d",
- "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"
+ "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472",
+ "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"
],
"markers": "python_version >= '3.8'",
- "version": "==2.2.1"
+ "version": "==2.2.2"
},
"wrapt": {
"hashes": [
diff --git a/dbrepo-analyse-service/lib/dbrepo-1.4.4-py3-none-any.whl b/dbrepo-analyse-service/lib/dbrepo-1.4.4-py3-none-any.whl
index 7e8fd7fca5aa6158bf57952f7f1050a08b331402..617969c3eb15926d932b7c0180bed51b9ef7052d 100644
Binary files a/dbrepo-analyse-service/lib/dbrepo-1.4.4-py3-none-any.whl and b/dbrepo-analyse-service/lib/dbrepo-1.4.4-py3-none-any.whl differ
diff --git a/dbrepo-analyse-service/lib/dbrepo-1.4.4.tar.gz b/dbrepo-analyse-service/lib/dbrepo-1.4.4.tar.gz
index 3e45d4513a31a1fda334ed9ad2c5cbad3803199a..9d1d5ae238baba6bc51db4d219a0d09b5aca1c51 100644
Binary files a/dbrepo-analyse-service/lib/dbrepo-1.4.4.tar.gz and b/dbrepo-analyse-service/lib/dbrepo-1.4.4.tar.gz differ
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SubsetServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SubsetServiceIntegrationTest.java
index aa30bc0580ff3b0aba950a1f1ea495b135280782..d13a37531b06e3572547b1f472e970ada78a7faa 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SubsetServiceIntegrationTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SubsetServiceIntegrationTest.java
@@ -95,6 +95,35 @@ public class SubsetServiceIntegrationTest extends AbstractUnitTest {
assertEquals(0.0, response.getResult().get(2).get("rainfall"));
}
+ @Test
+ public void execute_joinWithAlias_succeeds() throws QueryStoreInsertException, TableMalformedException, SQLException,
+ QueryNotFoundException, InterruptedException, UserNotFoundException, NotAllowedException,
+ RemoteUnavailableException, ServiceException, DatabaseNotFoundException {
+
+ /* pre-condition */
+ Thread.sleep(1000) /* wait for test container some more */;
+
+ /* mock */
+ when(metadataServiceGateway.getUserById(QUERY_1_CREATED_BY))
+ .thenReturn(QUERY_1_CREATOR);
+
+ /* test */
+ final QueryResultDto response = queryService.execute(DATABASE_1_PRIVILEGED_DTO, QUERY_7_STATEMENT, Instant.now(), USER_1_ID, 0L, 10L, null, null);
+ assertNotNull(response);
+ assertNotNull(response.getId());
+ assertNotNull(response.getHeaders());
+ assertEquals(5, response.getHeaders().size());
+ assertEquals(List.of(Map.of("id", 0), Map.of("date", 1), Map.of("location", 2), Map.of("lat", 3), Map.of("lng", 4)), response.getHeaders());
+ assertNotNull(response.getResult());
+ assertEquals(1, response.getResult().size());
+ /* row 0 */
+ assertEquals(BigInteger.valueOf(1L), response.getResult().get(0).get("id"));
+ assertEquals(Instant.ofEpochSecond(1228089600), response.getResult().get(0).get("date"));
+ assertEquals("Albury", response.getResult().get(0).get("location"));
+ assertEquals(-36.0653583, response.getResult().get(0).get("lat"));
+ assertEquals(146.9112214, response.getResult().get(0).get("lng"));
+ }
+
@Test
public void execute_oneResult_succeeds() throws QueryStoreInsertException, TableMalformedException, SQLException,
QueryNotFoundException, InterruptedException, UserNotFoundException, NotAllowedException,
diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java
index eea4d65320ed1002ebcff6c6120e3c372a2e0027..20a769f8ce8e8b8a5f4cdf1f4e42610e0e3f561d 100644
--- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java
+++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java
@@ -86,21 +86,24 @@ public class ViewServiceIntegrationTest extends AbstractUnitTest {
assertEquals(VIEW_2_ID, response.getId());
assertNotNull(response.getHeaders());
assertEquals(4, response.getHeaders().size());
- assertEquals(List.of(Map.of("date", 0), Map.of("location", 1), Map.of("mintemp", 2), Map.of("location", 3)), response.getHeaders());
+ assertEquals(List.of(Map.of("date", 0), Map.of("loc", 1), Map.of("mintemp", 2), Map.of("rainfall", 3)), response.getHeaders());
assertNotNull(response.getResult());
assertEquals(3, response.getResult().size());
/* row 0 */
assertEquals(Instant.ofEpochSecond(1228089600), response.getResult().get(0).get("date"));
- assertEquals("Albury", response.getResult().get(0).get("location"));
+ assertEquals("Albury", response.getResult().get(0).get("loc"));
assertEquals(13.4, response.getResult().get(0).get("mintemp"));
+ assertEquals(0.6, response.getResult().get(0).get("rainfall"));
/* row 1 */
assertEquals(Instant.ofEpochSecond(1228176000), response.getResult().get(1).get("date"));
- assertEquals("Albury", response.getResult().get(1).get("location"));
+ assertEquals("Albury", response.getResult().get(1).get("loc"));
assertEquals(7.4, response.getResult().get(1).get("mintemp"));
+ assertEquals(0.0, response.getResult().get(1).get("rainfall"));
/* row 2 */
assertEquals(Instant.ofEpochSecond(1228262400), response.getResult().get(2).get("date"));
- assertEquals("Albury", response.getResult().get(2).get("location"));
+ assertEquals("Albury", response.getResult().get(2).get("loc"));
assertEquals(12.9, response.getResult().get(2).get("mintemp"));
+ assertEquals(0.0, response.getResult().get(2).get("rainfall"));
}
@Test
diff --git a/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql b/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql
index 052f23adf8f5772a996d430877c8785d24651305..79755c97bf68fd822869277e3ad867c582f4e68c 100644
--- a/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql
+++ b/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql
@@ -66,7 +66,7 @@ VALUES ('2022-12-24 17:00:00', 10.0),
CREATE VIEW junit2 AS
(
-select `date`, `location`, `mintemp`, `rainfall`
+select `date`, `location` as loc, `mintemp`, `rainfall`
from `weather_aus`
where `location` = 'Albury');
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java
index 163ec5940fb36d2a228f61a745773cbdea9f0a8f..2e95fc34d28fdcea292f6336717ff4be2c5c03f6 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java
@@ -29,6 +29,7 @@ import com.google.common.hash.Hashing;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.parser.CCJSqlParserManager;
import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.select.*;
import org.jetbrains.annotations.NotNull;
import org.mapstruct.Mapper;
@@ -273,51 +274,63 @@ public interface DataMapper {
}
}
}
- final List<ColumnDto> allColumns = Stream.of(database.getViews()
- .stream()
- .map(ViewDto::getColumns)
- .flatMap(List::stream)
- .map(this::viewColumnDtoToColumnDto),
- database.getTables()
- .stream()
- .map(TableDto::getColumns)
- .flatMap(List::stream))
- .flatMap(i -> i)
+ final List<ColumnDto> allColumns = database.getTables()
+ .stream()
+ .map(TableDto::getColumns)
+ .flatMap(List::stream)
.toList();
log.trace("columns referenced in the from-clause and join-clause(s): {}", clauses);
/* Checking if all columns exist */
for (SelectItem clause : clauses) {
final SelectExpressionItem item = (SelectExpressionItem) clause;
final Column column = (Column) item.getExpression();
- final Optional<net.sf.jsqlparser.schema.Table> optional = fromItems.stream()
- .map(t -> (net.sf.jsqlparser.schema.Table) t)
- .filter(t -> {
- if (column.getTable() == null) {
- /* column does not reference a specific table, so there is only one table */
- final String tableName = ((net.sf.jsqlparser.schema.Table) fromItems.get(0)).getName().replace("`", "");
- return tableMatches(t, tableName);
- }
- final String tableName = column.getTable().getName().replace("`", "");
- return tableMatches(t, tableName);
- })
- .findFirst();
- if (optional.isEmpty()) {
- log.error("Failed to find table/view {} (with designator {})", column.getTable().getName(), column.getTable().getAlias());
- throw new JSQLParserException("Failed to find table/view " + column.getTable().getName() + " (with alias " + column.getTable().getAlias() + ")");
- }
final String columnName = column.getColumnName().replace("`", "");
- final String tableOrView = optional.get().getName().replace("`", "");
final List<ColumnDto> filteredColumns = allColumns.stream()
.filter(c -> (c.getAlias() != null && c.getAlias().equals(columnName)) || c.getInternalName().equals(columnName))
.toList();
- final Optional<ColumnDto> optionalColumn = filteredColumns.stream()
- .filter(c -> columnMatches(c, tableOrView))
- .findFirst();
- if (optionalColumn.isEmpty()) {
- log.error("Failed to find column with name {} of table/view {} in {}", columnName, tableOrView, filteredColumns.stream().map(c -> c.getTable().getInternalName() + "." + c.getInternalName()).toList());
- throw new JSQLParserException("Failed to find column with name " + columnName + " of table/view " + tableOrView);
+ String tableOrView = null;
+ for (Table t : fromItems.stream().map(t -> (net.sf.jsqlparser.schema.Table) t).toList()) {
+ if (column.getTable() == null) {
+ /* column does not reference a specific table, find out */
+ final List<String> filteredTables = filteredColumns.stream()
+ .map(c -> c.getTable().getInternalName())
+ .filter(table -> fromItems.stream().map(f -> (Table) f).anyMatch(otherTable -> otherTable.getName().replace("`", "").equals(table)))
+ .toList();
+ if (filteredTables.size() != 1) {
+ log.error("Failed to filter column {} to exactly one match: {}", columnName, filteredTables.stream().map(table -> table + "." + columnName).toList());
+ throw new JSQLParserException("Failed to filter column " + columnName + " to exactly one match");
+ }
+ if (tableMatches(t, filteredTables.get(0))) {
+ tableOrView = t.getName().replace("`", "");
+ break;
+ }
+ }
+ /* column references a specific table */
+ final String tableOrAlias = (t.getAlias() != null ? t.getAlias().getName() : column.getTable().getName())
+ .replace("`", "");
+ if (tableMatches(t, tableOrAlias)) {
+ tableOrView = t.getName().replace("`", "");
+ break;
+ }
+ }
+ if (tableOrView == null) {
+ log.error("Failed to find table/view {} (with designator {})", column.getTable().getName(), column.getTable().getAlias());
+ throw new JSQLParserException("Failed to find table/view " + column.getTable().getName() + " (with alias " + column.getTable().getAlias() + ")");
+ }
+ final String finalTableOrView = tableOrView;
+ final List<ColumnDto> selectColumns = filteredColumns.stream()
+ .filter(c -> c.getTable().getInternalName().equals(finalTableOrView))
+ .toList();
+ final ColumnDto resultColumn;
+ if (selectColumns.size() != 1) {
+ if (filteredColumns.size() != 1) {
+ log.error("Failed to filter column {} to exactly one match: {}", columnName, selectColumns.stream().map(c -> c.getTable().getInternalName() + "." + c.getInternalName()).toList());
+ throw new JSQLParserException("Failed to filter column " + columnName + " to exactly one match");
+ }
+ resultColumn = filteredColumns.get(0);
+ } else {
+ resultColumn = selectColumns.get(0);
}
- final ColumnDto resultColumn = optionalColumn.get();
if (item.getAlias() != null) {
resultColumn.setAlias(item.getAlias().getName().replace("`", ""));
}
@@ -330,14 +343,14 @@ public interface DataMapper {
return columns;
}
- default boolean tableMatches(net.sf.jsqlparser.schema.Table table, String otherTableName) {
+ default boolean tableMatches(net.sf.jsqlparser.schema.Table table, String tableOrDesignator) {
final String tableName = table.getName()
.trim()
.replace("`", "");
if (table.getAlias() == null) {
/* table does not have designator */
log.trace("table '{}' has no designator", tableName);
- return tableName.equals(otherTableName);
+ return tableName.equals(tableOrDesignator);
}
/* has designator */
final String designator = table.getAlias()
@@ -345,26 +358,7 @@ public interface DataMapper {
.trim()
.replace("`", "");
log.trace("table '{}' has designator {}", tableName, designator);
- return designator.equals(otherTableName);
- }
-
- default boolean columnMatches(ColumnDto column, String tableOrView) {
- if (column.getTable() != null && column.getTable().getInternalName().equals(tableOrView)) {
- log.trace("table '{}' found in column table", tableOrView);
- return true;
- }
- if (column.getViews() == null) {
- log.trace("table/view '{}' not found among column views: empty list", tableOrView);
- return false;
- }
- /* maybe matches one of the other views */
- final boolean found = column.getViews()
- .stream()
- .anyMatch(v -> v.getInternalName().equals(tableOrView));
- if (!found) {
- log.trace("table/view '{}' not found among column views: {}", tableOrView, column.getViews().stream().map(ViewDto::getInternalName).toList());
- }
- return found;
+ return designator.equals(tableOrDesignator);
}
default List<FromItem> fromItemToFromItems(FromItem data) throws JSQLParserException {
diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
index 8b43248271a686a3c23bceff02fc9d61ee801f9e..8cadf146b1e6acc6cac355d3890a492993a0fff5 100644
--- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
+++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java
@@ -353,12 +353,8 @@ public interface MariaDbMapper {
*/
default String selectDatasetRawQuery(String databaseName, String tableOrView, List<ColumnDto> columns,
Instant timestamp, Long size, Long page) {
- final int[] idx = new int[]{0};
- final StringBuilder statement = new StringBuilder("SELECT ");
- columns.forEach(column -> statement.append(idx[0]++ > 0 ? "," : "")
- .append("`")
- .append(column.getInternalName())
- .append("`"));
+ final StringBuilder statement = new StringBuilder("SELECT ")
+ .append(String.join(",", columns.stream().map(c -> "`" + c.getInternalName() + "`").toList()));
statement.append(" FROM `")
.append(databaseName)
.append("`.`")
diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/TableDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/TableDto.java
index 5764b311cacdd773e1e071718c2caf6db65977a7..a512cf8010195e5c107f35d8d2756251460bca62 100644
--- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/TableDto.java
+++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/TableDto.java
@@ -25,6 +25,7 @@ import java.util.UUID;
@AllArgsConstructor
@Jacksonized
@ToString
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class TableDto {
@NotNull
@@ -32,6 +33,7 @@ public class TableDto {
@NotNull
@JsonProperty("database_id")
+ @EqualsAndHashCode.Include
private Long tdbid;
@NotBlank
@@ -41,6 +43,7 @@ public class TableDto {
@NotBlank
@JsonProperty("internal_name")
@Schema(example = "air_quality")
+ @EqualsAndHashCode.Include
private String internalName;
@Schema
diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java
index 026d5d89b7dbb77a6bf4cd94b60448f8a73bc424..a506dbca82e9e64d4a0cec4fec12640452f56456 100644
--- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java
+++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java
@@ -25,6 +25,7 @@ import java.util.List;
@AllArgsConstructor
@Jacksonized
@ToString
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class ColumnDto {
@NotNull
@@ -52,6 +53,7 @@ public class ColumnDto {
@Size(max = 64)
@JsonProperty("internal_name")
@Schema(example = "mdb_date")
+ @EqualsAndHashCode.Include
private String internalName;
@Schema
@@ -122,6 +124,7 @@ public class ColumnDto {
@ToString.Exclude
@JsonIgnore
+ @EqualsAndHashCode.Include
private TableDto table;
@ToString.Exclude
diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java
index 8e628f710aa5eb13d12a48e7b1ded8214e17c9d2..fb7c80e0113838077f0a65826f636224440f9b94 100644
--- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java
+++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java
@@ -564,6 +564,9 @@ public interface MetadataMapper {
column.setDatabaseId(data.getDatabase().getId());
});
});
+ if (data.getConstraints().getChecks() == null || data.getConstraints().getChecks().isEmpty()) {
+ table.getConstraints().setChecks(new LinkedHashSet<>());
+ }
if (data.getIdentifiers() != null) {
table.setIdentifiers(new LinkedList<>(data.getIdentifiers()
.stream()
diff --git a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java
index 2d9a2f40a03a60b4650e8fa626d9ef17e7ab9eb4..b823e1c996fa3587df0ab6b7458dce27f5651f73 100644
--- a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java
+++ b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java
@@ -2602,6 +2602,8 @@ public abstract class BaseTest {
.resultNumber(2L)
.build();
+ public final static String QUERY_7_STATEMENT = "SELECT id, date, a.location, lat, lng FROM weather_aus a JOIN weather_location l on a.location = l.location WHERE date = '2008-12-01'";
+
public final static Long QUERY_4_ID = 4L;
public final static String QUERY_4_STATEMENT = "SELECT `id`, `value` FROM `mfcc`";
public final static String QUERY_4_QUERY_HASH = "df7da3801dfb5c191ff6711d79ce6455f3c09ec8323ce1ff7208ab85387263f5";
@@ -4929,13 +4931,13 @@ public abstract class BaseTest {
public final static Long VIEW_2_CONTAINER_ID = CONTAINER_1_ID;
public final static Long VIEW_2_DATABASE_ID = DATABASE_1_ID;
public final static Boolean VIEW_2_PUBLIC = true;
- public final static String VIEW_2_QUERY = "select `date`, `location` as loc, `location`, `rainfall`, `mintemp` from `weather_aus` where `location` = 'Albury'";
+ public final static String VIEW_2_QUERY = "select `date`, `location` as loc, `mintemp`, `rainfall` from `weather_aus` where `location` = 'Albury'";
public final static String VIEW_2_QUERY_HASH = "987fc946772ffb6d85060262dcb5df419692a1f6772ea995e3dedb53c191e984";
public final static List<ViewColumnDto> VIEW_2_COLUMNS_DTO = List.of(
ViewColumnDto.builder()
.id(4L)
- .name("Date")
+ .name("date")
.internalName("date")
.ordinalPosition(1)
.columnType(ColumnTypeDto.DATE)
@@ -4945,8 +4947,9 @@ public abstract class BaseTest {
.build(),
ViewColumnDto.builder()
.id(5L)
- .name("Location")
- .internalName("location")
+ .name("loc")
+ .internalName("loc")
+ .alias("loc")
.ordinalPosition(2)
.columnType(ColumnTypeDto.VARCHAR)
.size(255L)
@@ -4955,7 +4958,7 @@ public abstract class BaseTest {
.build(),
ViewColumnDto.builder()
.id(6L)
- .name("MinTemp")
+ .name("mintemp")
.internalName("mintemp")
.ordinalPosition(3)
.columnType(ColumnTypeDto.DECIMAL)
@@ -4966,11 +4969,12 @@ public abstract class BaseTest {
.build(),
ViewColumnDto.builder()
.id(7L)
- .name("Location")
- .internalName("location")
- .ordinalPosition(2)
- .columnType(ColumnTypeDto.VARCHAR)
- .size(255L)
+ .name("rainfall")
+ .internalName("rainfall")
+ .ordinalPosition(4)
+ .columnType(ColumnTypeDto.DECIMAL)
+ .size(10L)
+ .d(0L)
.isNullAllowed(true)
.autoGenerated(false)
.build()
diff --git a/dbrepo-search-service/Pipfile.lock b/dbrepo-search-service/Pipfile.lock
index 7117575adf21fb3bc4458a54cbb57c49c3e732ee..0ffce7c2f6e2888f0e4810affeb0ef24ddccb584 100644
--- a/dbrepo-search-service/Pipfile.lock
+++ b/dbrepo-search-service/Pipfile.lock
@@ -551,7 +551,7 @@
"sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da",
"sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"
],
- "markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
+ "markers": "python_version < '3.13' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
"version": "==3.0.3"
},
"gunicorn": {
@@ -796,45 +796,54 @@
},
"numpy": {
"hashes": [
- "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b",
- "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818",
- "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20",
- "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0",
- "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010",
- "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a",
- "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea",
- "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c",
- "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71",
- "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110",
- "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be",
- "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a",
- "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a",
- "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5",
- "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed",
- "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd",
- "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c",
- "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e",
- "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0",
- "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c",
- "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a",
- "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b",
- "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0",
- "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6",
- "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2",
- "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a",
- "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30",
- "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218",
- "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5",
- "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07",
- "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2",
- "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4",
- "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764",
- "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef",
- "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3",
- "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"
+ "sha256:04494f6ec467ccb5369d1808570ae55f6ed9b5809d7f035059000a37b8d7e86f",
+ "sha256:0a43f0974d501842866cc83471bdb0116ba0dffdbaac33ec05e6afed5b615238",
+ "sha256:0e50842b2295ba8414c8c1d9d957083d5dfe9e16828b37de883f51fc53c4016f",
+ "sha256:0ec84b9ba0654f3b962802edc91424331f423dcf5d5f926676e0150789cb3d95",
+ "sha256:17067d097ed036636fa79f6a869ac26df7db1ba22039d962422506640314933a",
+ "sha256:1cde1753efe513705a0c6d28f5884e22bdc30438bf0085c5c486cdaff40cd67a",
+ "sha256:1e72728e7501a450288fc8e1f9ebc73d90cfd4671ebbd631f3e7857c39bd16f2",
+ "sha256:2635dbd200c2d6faf2ef9a0d04f0ecc6b13b3cad54f7c67c61155138835515d2",
+ "sha256:2ce46fd0b8a0c947ae047d222f7136fc4d55538741373107574271bc00e20e8f",
+ "sha256:34f003cb88b1ba38cb9a9a4a3161c1604973d7f9d5552c38bc2f04f829536609",
+ "sha256:354f373279768fa5a584bac997de6a6c9bc535c482592d7a813bb0c09be6c76f",
+ "sha256:38ecb5b0582cd125f67a629072fed6f83562d9dd04d7e03256c9829bdec027ad",
+ "sha256:3e8e01233d57639b2e30966c63d36fcea099d17c53bf424d77f088b0f4babd86",
+ "sha256:3f6bed7f840d44c08ebdb73b1825282b801799e325bcbdfa6bc5c370e5aecc65",
+ "sha256:4554eb96f0fd263041baf16cf0881b3f5dafae7a59b1049acb9540c4d57bc8cb",
+ "sha256:46e161722e0f619749d1cd892167039015b2c2817296104487cd03ed4a955995",
+ "sha256:49d9f7d256fbc804391a7f72d4a617302b1afac1112fac19b6c6cec63fe7fe8a",
+ "sha256:4d2f62e55a4cd9c58c1d9a1c9edaedcd857a73cb6fda875bf79093f9d9086f85",
+ "sha256:5f64641b42b2429f56ee08b4f427a4d2daf916ec59686061de751a55aafa22e4",
+ "sha256:63b92c512d9dbcc37f9d81b123dec99fdb318ba38c8059afc78086fe73820275",
+ "sha256:6d7696c615765091cc5093f76fd1fa069870304beaccfd58b5dcc69e55ef49c1",
+ "sha256:79e843d186c8fb1b102bef3e2bc35ef81160ffef3194646a7fdd6a73c6b97196",
+ "sha256:821eedb7165ead9eebdb569986968b541f9908979c2da8a4967ecac4439bae3d",
+ "sha256:84554fc53daa8f6abf8e8a66e076aff6ece62de68523d9f665f32d2fc50fd66e",
+ "sha256:8d83bb187fb647643bd56e1ae43f273c7f4dbcdf94550d7938cfc32566756514",
+ "sha256:903703372d46bce88b6920a0cd86c3ad82dae2dbef157b5fc01b70ea1cfc430f",
+ "sha256:9416a5c2e92ace094e9f0082c5fd473502c91651fb896bc17690d6fc475128d6",
+ "sha256:9a1712c015831da583b21c5bfe15e8684137097969c6d22e8316ba66b5baabe4",
+ "sha256:9c27f0946a3536403efb0e1c28def1ae6730a72cd0d5878db38824855e3afc44",
+ "sha256:a356364941fb0593bb899a1076b92dfa2029f6f5b8ba88a14fd0984aaf76d0df",
+ "sha256:a7039a136017eaa92c1848152827e1424701532ca8e8967fe480fe1569dae581",
+ "sha256:acd3a644e4807e73b4e1867b769fbf1ce8c5d80e7caaef0d90dcdc640dfc9787",
+ "sha256:ad0c86f3455fbd0de6c31a3056eb822fc939f81b1618f10ff3406971893b62a5",
+ "sha256:b4c76e3d4c56f145d41b7b6751255feefae92edbc9a61e1758a98204200f30fc",
+ "sha256:b6f6a8f45d0313db07d6d1d37bd0b112f887e1369758a5419c0370ba915b3871",
+ "sha256:c5a59996dc61835133b56a32ebe4ef3740ea5bc19b3983ac60cc32be5a665d54",
+ "sha256:c73aafd1afca80afecb22718f8700b40ac7cab927b8abab3c3e337d70e10e5a2",
+ "sha256:cee6cc0584f71adefe2c908856ccc98702baf95ff80092e4ca46061538a2ba98",
+ "sha256:cef04d068f5fb0518a77857953193b6bb94809a806bd0a14983a8f12ada060c9",
+ "sha256:cf5d1c9e6837f8af9f92b6bd3e86d513cdc11f60fd62185cc49ec7d1aba34864",
+ "sha256:e61155fae27570692ad1d327e81c6cf27d535a5d7ef97648a17d922224b216de",
+ "sha256:e7f387600d424f91576af20518334df3d97bc76a300a755f9a8d6e4f5cadd289",
+ "sha256:ed08d2703b5972ec736451b818c2eb9da80d66c3e84aed1deeb0c345fefe461b",
+ "sha256:fbd6acc766814ea6443628f4e6751d0da6593dae29c08c0b2606164db026970c",
+ "sha256:feff59f27338135776f6d4e2ec7aeeac5d5f7a08a83e80869121ef8164b74af9"
],
"markers": "python_version == '3.11'",
- "version": "==1.26.4"
+ "version": "==2.0.0"
},
"opensearch-py": {
"hashes": [
@@ -1263,58 +1272,58 @@
},
"sqlalchemy": {
"hashes": [
- "sha256:0094c5dc698a5f78d3d1539853e8ecec02516b62b8223c970c86d44e7a80f6c7",
- "sha256:0138c5c16be3600923fa2169532205d18891b28afa817cb49b50e08f62198bb8",
- "sha256:0a089e218654e740a41388893e090d2e2c22c29028c9d1353feb38638820bbeb",
- "sha256:0b3f4c438e37d22b83e640f825ef0f37b95db9aa2d68203f2c9549375d0b2260",
- "sha256:16863e2b132b761891d6c49f0a0f70030e0bcac4fd208117f6b7e053e68668d0",
- "sha256:1f9a727312ff6ad5248a4367358e2cf7e625e98b1028b1d7ab7b806b7d757513",
- "sha256:2383146973a15435e4717f94c7509982770e3e54974c71f76500a0136f22810b",
- "sha256:2753743c2afd061bb95a61a51bbb6a1a11ac1c44292fad898f10c9839a7f75b2",
- "sha256:296230899df0b77dec4eb799bcea6fbe39a43707ce7bb166519c97b583cfcab3",
- "sha256:2a4f4da89c74435f2bc61878cd08f3646b699e7d2eba97144030d1be44e27584",
- "sha256:2b1708916730f4830bc69d6f49d37f7698b5bd7530aca7f04f785f8849e95255",
- "sha256:2ecabd9ccaa6e914e3dbb2aa46b76dede7eadc8cbf1b8083c94d936bcd5ffb49",
- "sha256:311710f9a2ee235f1403537b10c7687214bb1f2b9ebb52702c5aa4a77f0b3af7",
- "sha256:37a4b4fb0dd4d2669070fb05b8b8824afd0af57587393015baee1cf9890242d9",
- "sha256:3a365eda439b7a00732638f11072907c1bc8e351c7665e7e5da91b169af794af",
- "sha256:3b48154678e76445c7ded1896715ce05319f74b1e73cf82d4f8b59b46e9c0ddc",
- "sha256:3b69e934f0f2b677ec111b4d83f92dc1a3210a779f69bf905273192cf4ed433e",
- "sha256:3cb5a646930c5123f8461f6468901573f334c2c63c795b9af350063a736d0134",
- "sha256:408f8b0e2c04677e9c93f40eef3ab22f550fecb3011b187f66a096395ff3d9fd",
- "sha256:40ad017c672c00b9b663fcfcd5f0864a0a97828e2ee7ab0c140dc84058d194cf",
- "sha256:5a79d65395ac5e6b0c2890935bad892eabb911c4aa8e8015067ddb37eea3d56c",
- "sha256:5a8e3b0a7e09e94be7510d1661339d6b52daf202ed2f5b1f9f48ea34ee6f2d57",
- "sha256:69c9db1ce00e59e8dd09d7bae852a9add716efdc070a3e2068377e6ff0d6fdaa",
- "sha256:7108d569d3990c71e26a42f60474b4c02c8586c4681af5fd67e51a044fdea86a",
- "sha256:77d2edb1f54aff37e3318f611637171e8ec71472f1fdc7348b41dcb226f93d90",
- "sha256:7d74336c65705b986d12a7e337ba27ab2b9d819993851b140efdf029248e818e",
- "sha256:8409de825f2c3b62ab15788635ccaec0c881c3f12a8af2b12ae4910a0a9aeef6",
- "sha256:955991a09f0992c68a499791a753523f50f71a6885531568404fa0f231832aa0",
- "sha256:99650e9f4cf3ad0d409fed3eec4f071fadd032e9a5edc7270cd646a26446feeb",
- "sha256:9a5baf9267b752390252889f0c802ea13b52dfee5e369527da229189b8bd592e",
- "sha256:a0ef36b28534f2a5771191be6edb44cc2673c7b2edf6deac6562400288664221",
- "sha256:a1429a4b0f709f19ff3b0cf13675b2b9bfa8a7e79990003207a011c0db880a13",
- "sha256:a7bfc726d167f425d4c16269a9a10fe8630ff6d14b683d588044dcef2d0f6be7",
- "sha256:a943d297126c9230719c27fcbbeab57ecd5d15b0bd6bfd26e91bfcfe64220621",
- "sha256:ae8c62fe2480dd61c532ccafdbce9b29dacc126fe8be0d9a927ca3e699b9491a",
- "sha256:b60203c63e8f984df92035610c5fb76d941254cf5d19751faab7d33b21e5ddc0",
- "sha256:b6bf767d14b77f6a18b6982cbbf29d71bede087edae495d11ab358280f304d8e",
- "sha256:b6c7ec2b1f4969fc19b65b7059ed00497e25f54069407a8701091beb69e591a5",
- "sha256:bba002a9447b291548e8d66fd8c96a6a7ed4f2def0bb155f4f0a1309fd2735d5",
- "sha256:bc0c53579650a891f9b83fa3cecd4e00218e071d0ba00c4890f5be0c34887ed3",
- "sha256:c4f61ada6979223013d9ab83a3ed003ded6959eae37d0d685db2c147e9143797",
- "sha256:c62d401223f468eb4da32627bffc0c78ed516b03bb8a34a58be54d618b74d472",
- "sha256:e42203d8d20dc704604862977b1470a122e4892791fe3ed165f041e4bf447a1b",
- "sha256:edc16a50f5e1b7a06a2dcc1f2205b0b961074c123ed17ebda726f376a5ab0953",
- "sha256:efedba7e13aa9a6c8407c48facfdfa108a5a4128e35f4c68f20c3407e4376aa9",
- "sha256:f1dc3eabd8c0232ee8387fbe03e0a62220a6f089e278b1f0aaf5e2d6210741ad",
- "sha256:f69e4c756ee2686767eb80f94c0125c8b0a0b87ede03eacc5c8ae3b54b99dc46",
- "sha256:f7703c2010355dd28f53deb644a05fc30f796bd8598b43f0ba678878780b6e4c",
- "sha256:fa561138a64f949f3e889eb9ab8c58e1504ab351d6cf55259dc4c248eaa19da6"
+ "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96",
+ "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396",
+ "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb",
+ "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac",
+ "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704",
+ "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca",
+ "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe",
+ "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808",
+ "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4",
+ "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42",
+ "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc",
+ "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c",
+ "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238",
+ "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f",
+ "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740",
+ "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d",
+ "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327",
+ "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4",
+ "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e",
+ "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911",
+ "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac",
+ "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c",
+ "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e",
+ "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58",
+ "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4",
+ "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88",
+ "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be",
+ "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005",
+ "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc",
+ "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443",
+ "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1",
+ "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484",
+ "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1",
+ "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67",
+ "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203",
+ "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05",
+ "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227",
+ "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3",
+ "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449",
+ "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109",
+ "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b",
+ "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3",
+ "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9",
+ "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c",
+ "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa",
+ "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0",
+ "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298",
+ "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a",
+ "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"
],
"markers": "python_version >= '3.7'",
- "version": "==2.0.30"
+ "version": "==2.0.31"
},
"sqlalchemy-utils": {
"hashes": [
@@ -1372,11 +1381,11 @@
},
"urllib3": {
"hashes": [
- "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d",
- "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"
+ "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472",
+ "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"
],
"markers": "python_version >= '3.10'",
- "version": "==2.2.1"
+ "version": "==2.2.2"
},
"werkzeug": {
"hashes": [
diff --git a/dbrepo-search-service/init/Dockerfile b/dbrepo-search-service/init/Dockerfile
index 01a27175316bbded7802c6178a157388d56ced30..ebde913dbd9e013e9591fa4135e919a78fbbbd26 100644
--- a/dbrepo-search-service/init/Dockerfile
+++ b/dbrepo-search-service/init/Dockerfile
@@ -6,6 +6,8 @@ WORKDIR /home/alpine
COPY Pipfile Pipfile.lock ./
+COPY ./lib ./lib
+
RUN pip install pipenv && \
pipenv install gunicorn && \
pipenv install --system --deploy
diff --git a/dbrepo-search-service/init/Pipfile b/dbrepo-search-service/init/Pipfile
index 8676f227dc6970ce9ceda2633df0d9bc496a1959..517796af748f40cf55f52bac420a000c04c11b23 100644
--- a/dbrepo-search-service/init/Pipfile
+++ b/dbrepo-search-service/init/Pipfile
@@ -9,7 +9,7 @@ opensearch-py = "~=2.2"
python-dotenv = "~=1.0"
testcontainers-opensearch = "*"
pytest = "*"
-dbrepo = "1.4.3rc3"
+dbrepo = {path = "./lib/dbrepo-1.4.4.tar.gz"}
[dev-packages]
coverage = "*"
diff --git a/dbrepo-search-service/init/Pipfile.lock b/dbrepo-search-service/init/Pipfile.lock
index d88be6346de6fafdb47abe238384b2970ddfe546..37ffb270abdd1104ad58eb249b63b610bf88606f 100644
--- a/dbrepo-search-service/init/Pipfile.lock
+++ b/dbrepo-search-service/init/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "d54312bd3fff7b1b422c47cd63404ce8b48233b17baaaf7278b492989b3a9a77"
+ "sha256": "b12551e0f7592ebabd1eb3ad3efe9e7304c4dcee4fcb065afa88308bff71855d"
},
"pipfile-spec": 6,
"requires": {
@@ -108,11 +108,11 @@
},
"annotated-types": {
"hashes": [
- "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43",
- "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"
+ "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53",
+ "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"
],
"markers": "python_version >= '3.8'",
- "version": "==0.6.0"
+ "version": "==0.7.0"
},
"attrs": {
"hashes": [
@@ -132,11 +132,11 @@
},
"certifi": {
"hashes": [
- "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f",
- "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"
+ "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516",
+ "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"
],
"markers": "python_version >= '3.6'",
- "version": "==2024.2.2"
+ "version": "==2024.6.2"
},
"charset-normalizer": {
"hashes": [
@@ -244,19 +244,24 @@
},
"dbrepo": {
"hashes": [
- "sha256:012c846399ac031ee9cf6c9f6e17bc209bcae86b121a8cb99cd889e5c5e56ad1",
- "sha256:99a0b512e0a78c67fa919e82e2405b62f3585e1a8f680bc1b7c7108820b32aa8"
+ "sha256:d445f1c5a361eae8f3538a395828a297dd999ab36e5bbca0ad1cca9d093b8030"
],
- "index": "pypi",
- "version": "==1.4.3rc3"
+ "path": "./lib/dbrepo-1.4.4.tar.gz",
+ "version": "==1.4.4"
},
"docker": {
"hashes": [
- "sha256:12ba681f2777a0ad28ffbcc846a69c31b4dfd9752b47eb425a274ee269c5e14b",
- "sha256:323736fb92cd9418fc5e7133bc953e11a9da04f4483f828b527db553f1e7e5a3"
+ "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c",
+ "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"
],
"markers": "python_version >= '3.8'",
- "version": "==7.0.0"
+ "version": "==7.1.0"
+ },
+ "events": {
+ "hashes": [
+ "sha256:a7286af378ba3e46640ac9825156c93bdba7502174dd696090fdfcd4d80a1abd"
+ ],
+ "version": "==0.5"
},
"flask": {
"hashes": [
@@ -545,61 +550,70 @@
},
"numpy": {
"hashes": [
- "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b",
- "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818",
- "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20",
- "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0",
- "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010",
- "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a",
- "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea",
- "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c",
- "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71",
- "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110",
- "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be",
- "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a",
- "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a",
- "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5",
- "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed",
- "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd",
- "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c",
- "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e",
- "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0",
- "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c",
- "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a",
- "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b",
- "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0",
- "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6",
- "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2",
- "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a",
- "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30",
- "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218",
- "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5",
- "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07",
- "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2",
- "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4",
- "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764",
- "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef",
- "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3",
- "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"
+ "sha256:04494f6ec467ccb5369d1808570ae55f6ed9b5809d7f035059000a37b8d7e86f",
+ "sha256:0a43f0974d501842866cc83471bdb0116ba0dffdbaac33ec05e6afed5b615238",
+ "sha256:0e50842b2295ba8414c8c1d9d957083d5dfe9e16828b37de883f51fc53c4016f",
+ "sha256:0ec84b9ba0654f3b962802edc91424331f423dcf5d5f926676e0150789cb3d95",
+ "sha256:17067d097ed036636fa79f6a869ac26df7db1ba22039d962422506640314933a",
+ "sha256:1cde1753efe513705a0c6d28f5884e22bdc30438bf0085c5c486cdaff40cd67a",
+ "sha256:1e72728e7501a450288fc8e1f9ebc73d90cfd4671ebbd631f3e7857c39bd16f2",
+ "sha256:2635dbd200c2d6faf2ef9a0d04f0ecc6b13b3cad54f7c67c61155138835515d2",
+ "sha256:2ce46fd0b8a0c947ae047d222f7136fc4d55538741373107574271bc00e20e8f",
+ "sha256:34f003cb88b1ba38cb9a9a4a3161c1604973d7f9d5552c38bc2f04f829536609",
+ "sha256:354f373279768fa5a584bac997de6a6c9bc535c482592d7a813bb0c09be6c76f",
+ "sha256:38ecb5b0582cd125f67a629072fed6f83562d9dd04d7e03256c9829bdec027ad",
+ "sha256:3e8e01233d57639b2e30966c63d36fcea099d17c53bf424d77f088b0f4babd86",
+ "sha256:3f6bed7f840d44c08ebdb73b1825282b801799e325bcbdfa6bc5c370e5aecc65",
+ "sha256:4554eb96f0fd263041baf16cf0881b3f5dafae7a59b1049acb9540c4d57bc8cb",
+ "sha256:46e161722e0f619749d1cd892167039015b2c2817296104487cd03ed4a955995",
+ "sha256:49d9f7d256fbc804391a7f72d4a617302b1afac1112fac19b6c6cec63fe7fe8a",
+ "sha256:4d2f62e55a4cd9c58c1d9a1c9edaedcd857a73cb6fda875bf79093f9d9086f85",
+ "sha256:5f64641b42b2429f56ee08b4f427a4d2daf916ec59686061de751a55aafa22e4",
+ "sha256:63b92c512d9dbcc37f9d81b123dec99fdb318ba38c8059afc78086fe73820275",
+ "sha256:6d7696c615765091cc5093f76fd1fa069870304beaccfd58b5dcc69e55ef49c1",
+ "sha256:79e843d186c8fb1b102bef3e2bc35ef81160ffef3194646a7fdd6a73c6b97196",
+ "sha256:821eedb7165ead9eebdb569986968b541f9908979c2da8a4967ecac4439bae3d",
+ "sha256:84554fc53daa8f6abf8e8a66e076aff6ece62de68523d9f665f32d2fc50fd66e",
+ "sha256:8d83bb187fb647643bd56e1ae43f273c7f4dbcdf94550d7938cfc32566756514",
+ "sha256:903703372d46bce88b6920a0cd86c3ad82dae2dbef157b5fc01b70ea1cfc430f",
+ "sha256:9416a5c2e92ace094e9f0082c5fd473502c91651fb896bc17690d6fc475128d6",
+ "sha256:9a1712c015831da583b21c5bfe15e8684137097969c6d22e8316ba66b5baabe4",
+ "sha256:9c27f0946a3536403efb0e1c28def1ae6730a72cd0d5878db38824855e3afc44",
+ "sha256:a356364941fb0593bb899a1076b92dfa2029f6f5b8ba88a14fd0984aaf76d0df",
+ "sha256:a7039a136017eaa92c1848152827e1424701532ca8e8967fe480fe1569dae581",
+ "sha256:acd3a644e4807e73b4e1867b769fbf1ce8c5d80e7caaef0d90dcdc640dfc9787",
+ "sha256:ad0c86f3455fbd0de6c31a3056eb822fc939f81b1618f10ff3406971893b62a5",
+ "sha256:b4c76e3d4c56f145d41b7b6751255feefae92edbc9a61e1758a98204200f30fc",
+ "sha256:b6f6a8f45d0313db07d6d1d37bd0b112f887e1369758a5419c0370ba915b3871",
+ "sha256:c5a59996dc61835133b56a32ebe4ef3740ea5bc19b3983ac60cc32be5a665d54",
+ "sha256:c73aafd1afca80afecb22718f8700b40ac7cab927b8abab3c3e337d70e10e5a2",
+ "sha256:cee6cc0584f71adefe2c908856ccc98702baf95ff80092e4ca46061538a2ba98",
+ "sha256:cef04d068f5fb0518a77857953193b6bb94809a806bd0a14983a8f12ada060c9",
+ "sha256:cf5d1c9e6837f8af9f92b6bd3e86d513cdc11f60fd62185cc49ec7d1aba34864",
+ "sha256:e61155fae27570692ad1d327e81c6cf27d535a5d7ef97648a17d922224b216de",
+ "sha256:e7f387600d424f91576af20518334df3d97bc76a300a755f9a8d6e4f5cadd289",
+ "sha256:ed08d2703b5972ec736451b818c2eb9da80d66c3e84aed1deeb0c345fefe461b",
+ "sha256:fbd6acc766814ea6443628f4e6751d0da6593dae29c08c0b2606164db026970c",
+ "sha256:feff59f27338135776f6d4e2ec7aeeac5d5f7a08a83e80869121ef8164b74af9"
],
"markers": "python_version == '3.11'",
- "version": "==1.26.4"
+ "version": "==2.0.0"
},
"opensearch-py": {
"hashes": [
- "sha256:0dde4ac7158a717d92a8cd81964cb99705a4b80bcf9258ba195b9a9f23f5226d",
- "sha256:cf093a40e272b60663f20417fc1264ac724dcf1e03c1a4542a6b44835b1e6c49"
+ "sha256:0b7c27e8ed84c03c99558406927b6161f186a72502ca6d0325413d8e5523ba96",
+ "sha256:b6e78b685dd4e9c016d7a4299cf1de69e299c88322e3f81c716e6e23fe5683c1"
],
"index": "pypi",
- "version": "==2.5.0"
+ "version": "==2.6.0"
},
"packaging": {
"hashes": [
- "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5",
- "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"
+ "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002",
+ "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
],
- "markers": "python_version >= '3.7'",
- "version": "==24.0"
+ "markers": "python_version >= '3.8'",
+ "version": "==24.1"
},
"pandas": {
"hashes": [
@@ -654,104 +668,104 @@
},
"pydantic": {
"hashes": [
- "sha256:e029badca45266732a9a79898a15ae2e8b14840b1eabbb25844be28f0b33f3d5",
- "sha256:e9dbb5eada8abe4d9ae5f46b9939aead650cd2b68f249bb3a8139dbe125803cc"
+ "sha256:0c84efd9548d545f63ac0060c1e4d39bb9b14db8b3c0652338aecc07b5adec52",
+ "sha256:ee8538d41ccb9c0a9ad3e0e5f07bf15ed8015b481ced539a1759d8cc89ae90d0"
],
"markers": "python_version >= '3.8'",
- "version": "==2.7.1"
+ "version": "==2.7.4"
},
"pydantic-core": {
"hashes": [
- "sha256:0098300eebb1c837271d3d1a2cd2911e7c11b396eac9661655ee524a7f10587b",
- "sha256:042473b6280246b1dbf530559246f6842b56119c2926d1e52b631bdc46075f2a",
- "sha256:05b7133a6e6aeb8df37d6f413f7705a37ab4031597f64ab56384c94d98fa0e90",
- "sha256:0680b1f1f11fda801397de52c36ce38ef1c1dc841a0927a94f226dea29c3ae3d",
- "sha256:0d69b4c2f6bb3e130dba60d34c0845ba31b69babdd3f78f7c0c8fae5021a253e",
- "sha256:1404c69d6a676245199767ba4f633cce5f4ad4181f9d0ccb0577e1f66cf4c46d",
- "sha256:182245ff6b0039e82b6bb585ed55a64d7c81c560715d1bad0cbad6dfa07b4027",
- "sha256:1a388a77e629b9ec814c1b1e6b3b595fe521d2cdc625fcca26fbc2d44c816804",
- "sha256:1d90c3265ae107f91a4f279f4d6f6f1d4907ac76c6868b27dc7fb33688cfb347",
- "sha256:20aca1e2298c56ececfd8ed159ae4dde2df0781988c97ef77d5c16ff4bd5b400",
- "sha256:219da3f096d50a157f33645a1cf31c0ad1fe829a92181dd1311022f986e5fbe3",
- "sha256:22057013c8c1e272eb8d0eebc796701167d8377441ec894a8fed1af64a0bf399",
- "sha256:223ee893d77a310a0391dca6df00f70bbc2f36a71a895cecd9a0e762dc37b349",
- "sha256:224c421235f6102e8737032483f43c1a8cfb1d2f45740c44166219599358c2cd",
- "sha256:2334ce8c673ee93a1d6a65bd90327588387ba073c17e61bf19b4fd97d688d63c",
- "sha256:269322dcc3d8bdb69f054681edff86276b2ff972447863cf34c8b860f5188e2e",
- "sha256:2728b01246a3bba6de144f9e3115b532ee44bd6cf39795194fb75491824a1413",
- "sha256:2b8ed04b3582771764538f7ee7001b02e1170223cf9b75dff0bc698fadb00cf3",
- "sha256:2e29d20810dfc3043ee13ac7d9e25105799817683348823f305ab3f349b9386e",
- "sha256:36789b70d613fbac0a25bb07ab3d9dba4d2e38af609c020cf4d888d165ee0bf3",
- "sha256:390193c770399861d8df9670fb0d1874f330c79caaca4642332df7c682bf6b91",
- "sha256:3a6515ebc6e69d85502b4951d89131ca4e036078ea35533bb76327f8424531ce",
- "sha256:3f9a801e7c8f1ef8718da265bba008fa121243dfe37c1cea17840b0944dfd72c",
- "sha256:43f0f463cf89ace478de71a318b1b4f05ebc456a9b9300d027b4b57c1a2064fb",
- "sha256:4456f2dca97c425231d7315737d45239b2b51a50dc2b6f0c2bb181fce6207664",
- "sha256:470b94480bb5ee929f5acba6995251ada5e059a5ef3e0dfc63cca287283ebfa6",
- "sha256:4774f3184d2ef3e14e8693194f661dea5a4d6ca4e3dc8e39786d33a94865cefd",
- "sha256:4b4356d3538c3649337df4074e81b85f0616b79731fe22dd11b99499b2ebbdf3",
- "sha256:553ef617b6836fc7e4df130bb851e32fe357ce36336d897fd6646d6058d980af",
- "sha256:6132dd3bd52838acddca05a72aafb6eab6536aa145e923bb50f45e78b7251043",
- "sha256:6a46e22a707e7ad4484ac9ee9f290f9d501df45954184e23fc29408dfad61350",
- "sha256:6e5c584d357c4e2baf0ff7baf44f4994be121e16a2c88918a5817331fc7599d7",
- "sha256:75250dbc5290e3f1a0f4618db35e51a165186f9034eff158f3d490b3fed9f8a0",
- "sha256:75f7e9488238e920ab6204399ded280dc4c307d034f3924cd7f90a38b1829563",
- "sha256:78363590ef93d5d226ba21a90a03ea89a20738ee5b7da83d771d283fd8a56761",
- "sha256:7ca4ae5a27ad7a4ee5170aebce1574b375de390bc01284f87b18d43a3984df72",
- "sha256:800d60565aec896f25bc3cfa56d2277d52d5182af08162f7954f938c06dc4ee3",
- "sha256:82d5d4d78e4448683cb467897fe24e2b74bb7b973a541ea1dcfec1d3cbce39fb",
- "sha256:852e966fbd035a6468fc0a3496589b45e2208ec7ca95c26470a54daed82a0788",
- "sha256:868649da93e5a3d5eacc2b5b3b9235c98ccdbfd443832f31e075f54419e1b96b",
- "sha256:886eec03591b7cf058467a70a87733b35f44707bd86cf64a615584fd72488b7c",
- "sha256:8b172601454f2d7701121bbec3425dd71efcb787a027edf49724c9cefc14c038",
- "sha256:95b9d5e72481d3780ba3442eac863eae92ae43a5f3adb5b4d0a1de89d42bb250",
- "sha256:98758d627ff397e752bc339272c14c98199c613f922d4a384ddc07526c86a2ec",
- "sha256:997abc4df705d1295a42f95b4eec4950a37ad8ae46d913caeee117b6b198811c",
- "sha256:9b5155ff768083cb1d62f3e143b49a8a3432e6789a3abee8acd005c3c7af1c74",
- "sha256:9e08e867b306f525802df7cd16c44ff5ebbe747ff0ca6cf3fde7f36c05a59a81",
- "sha256:9fdad8e35f278b2c3eb77cbdc5c0a49dada440657bf738d6905ce106dc1de439",
- "sha256:a1874c6dd4113308bd0eb568418e6114b252afe44319ead2b4081e9b9521fe75",
- "sha256:a8309f67285bdfe65c372ea3722b7a5642680f3dba538566340a9d36e920b5f0",
- "sha256:ae0a8a797a5e56c053610fa7be147993fe50960fa43609ff2a9552b0e07013e8",
- "sha256:b14d82cdb934e99dda6d9d60dc84a24379820176cc4a0d123f88df319ae9c150",
- "sha256:b1bd7e47b1558ea872bd16c8502c414f9e90dcf12f1395129d7bb42a09a95438",
- "sha256:b3ef08e20ec49e02d5c6717a91bb5af9b20f1805583cb0adfe9ba2c6b505b5ae",
- "sha256:b89ed9eb7d616ef5714e5590e6cf7f23b02d0d539767d33561e3675d6f9e3857",
- "sha256:c4fcf5cd9c4b655ad666ca332b9a081112cd7a58a8b5a6ca7a3104bc950f2038",
- "sha256:c6fdc8627910eed0c01aed6a390a252fe3ea6d472ee70fdde56273f198938374",
- "sha256:c9bd70772c720142be1020eac55f8143a34ec9f82d75a8e7a07852023e46617f",
- "sha256:ca7b0c1f1c983e064caa85f3792dd2fe3526b3505378874afa84baf662e12241",
- "sha256:cbca948f2d14b09d20268cda7b0367723d79063f26c4ffc523af9042cad95592",
- "sha256:cc1cfd88a64e012b74e94cd00bbe0f9c6df57049c97f02bb07d39e9c852e19a4",
- "sha256:ccdd111c03bfd3666bd2472b674c6899550e09e9f298954cfc896ab92b5b0e6d",
- "sha256:cfeecd1ac6cc1fb2692c3d5110781c965aabd4ec5d32799773ca7b1456ac636b",
- "sha256:d4d938ec0adf5167cb335acb25a4ee69a8107e4984f8fbd2e897021d9e4ca21b",
- "sha256:d7d904828195733c183d20a54230c0df0eb46ec746ea1a666730787353e87182",
- "sha256:d91cb5ea8b11607cc757675051f61b3d93f15eca3cefb3e6c704a5d6e8440f4e",
- "sha256:d9319e499827271b09b4e411905b24a426b8fb69464dfa1696258f53a3334641",
- "sha256:e0e8b1be28239fc64a88a8189d1df7fad8be8c1ae47fcc33e43d4be15f99cc70",
- "sha256:e18609ceaa6eed63753037fc06ebb16041d17d28199ae5aba0052c51449650a9",
- "sha256:e1b395e58b10b73b07b7cf740d728dd4ff9365ac46c18751bf8b3d8cca8f625a",
- "sha256:e23ec367a948b6d812301afc1b13f8094ab7b2c280af66ef450efc357d2ae543",
- "sha256:e25add29b8f3b233ae90ccef2d902d0ae0432eb0d45370fe315d1a5cf231004b",
- "sha256:e6dac87ddb34aaec85f873d737e9d06a3555a1cc1a8e0c44b7f8d5daeb89d86f",
- "sha256:ef26c9e94a8c04a1b2924149a9cb081836913818e55681722d7f29af88fe7b38",
- "sha256:eff2de745698eb46eeb51193a9f41d67d834d50e424aef27df2fcdee1b153845",
- "sha256:f0a21cbaa69900cbe1a2e7cad2aa74ac3cf21b10c3efb0fa0b80305274c0e8a2",
- "sha256:f459a5ce8434614dfd39bbebf1041952ae01da6bed9855008cb33b875cb024c0",
- "sha256:f93a8a2e3938ff656a7c1bc57193b1319960ac015b6e87d76c76bf14fe0244b4",
- "sha256:fb2bd7be70c0fe4dfd32c951bc813d9fe6ebcbfdd15a07527796c8204bd36242"
+ "sha256:01dd777215e2aa86dfd664daed5957704b769e726626393438f9c87690ce78c3",
+ "sha256:0eb2a4f660fcd8e2b1c90ad566db2b98d7f3f4717c64fe0a83e0adb39766d5b8",
+ "sha256:0fbbdc827fe5e42e4d196c746b890b3d72876bdbf160b0eafe9f0334525119c8",
+ "sha256:123c3cec203e3f5ac7b000bd82235f1a3eced8665b63d18be751f115588fea30",
+ "sha256:14601cdb733d741b8958224030e2bfe21a4a881fb3dd6fbb21f071cabd48fa0a",
+ "sha256:18f469a3d2a2fdafe99296a87e8a4c37748b5080a26b806a707f25a902c040a8",
+ "sha256:19894b95aacfa98e7cb093cd7881a0c76f55731efad31073db4521e2b6ff5b7d",
+ "sha256:1b4de2e51bbcb61fdebd0ab86ef28062704f62c82bbf4addc4e37fa4b00b7cbc",
+ "sha256:1d886dc848e60cb7666f771e406acae54ab279b9f1e4143babc9c2258213daa2",
+ "sha256:1f4d26ceb5eb9eed4af91bebeae4b06c3fb28966ca3a8fb765208cf6b51102ab",
+ "sha256:21a5e440dbe315ab9825fcd459b8814bb92b27c974cbc23c3e8baa2b76890077",
+ "sha256:293afe532740370aba8c060882f7d26cfd00c94cae32fd2e212a3a6e3b7bc15e",
+ "sha256:2f5966897e5461f818e136b8451d0551a2e77259eb0f73a837027b47dc95dab9",
+ "sha256:2fd41f6eff4c20778d717af1cc50eca52f5afe7805ee530a4fbd0bae284f16e9",
+ "sha256:2fdf2156aa3d017fddf8aea5adfba9f777db1d6022d392b682d2a8329e087cef",
+ "sha256:3c40d4eaad41f78e3bbda31b89edc46a3f3dc6e171bf0ecf097ff7a0ffff7cb1",
+ "sha256:43d447dd2ae072a0065389092a231283f62d960030ecd27565672bd40746c507",
+ "sha256:44a688331d4a4e2129140a8118479443bd6f1905231138971372fcde37e43528",
+ "sha256:44c7486a4228413c317952e9d89598bcdfb06399735e49e0f8df643e1ccd0558",
+ "sha256:44cd83ab6a51da80fb5adbd9560e26018e2ac7826f9626bc06ca3dc074cd198b",
+ "sha256:46387e38bd641b3ee5ce247563b60c5ca098da9c56c75c157a05eaa0933ed154",
+ "sha256:4701b19f7e3a06ea655513f7938de6f108123bf7c86bbebb1196eb9bd35cf724",
+ "sha256:4748321b5078216070b151d5271ef3e7cc905ab170bbfd27d5c83ee3ec436695",
+ "sha256:4b06beb3b3f1479d32befd1f3079cc47b34fa2da62457cdf6c963393340b56e9",
+ "sha256:4d0dcc59664fcb8974b356fe0a18a672d6d7cf9f54746c05f43275fc48636851",
+ "sha256:4e99bc050fe65c450344421017f98298a97cefc18c53bb2f7b3531eb39bc7805",
+ "sha256:509daade3b8649f80d4e5ff21aa5673e4ebe58590b25fe42fac5f0f52c6f034a",
+ "sha256:51991a89639a912c17bef4b45c87bd83593aee0437d8102556af4885811d59f5",
+ "sha256:53db086f9f6ab2b4061958d9c276d1dbe3690e8dd727d6abf2321d6cce37fa94",
+ "sha256:564d7922e4b13a16b98772441879fcdcbe82ff50daa622d681dd682175ea918c",
+ "sha256:574d92eac874f7f4db0ca653514d823a0d22e2354359d0759e3f6a406db5d55d",
+ "sha256:578e24f761f3b425834f297b9935e1ce2e30f51400964ce4801002435a1b41ef",
+ "sha256:59ff3e89f4eaf14050c8022011862df275b552caef8082e37b542b066ce1ff26",
+ "sha256:5f09baa656c904807e832cf9cce799c6460c450c4ad80803517032da0cd062e2",
+ "sha256:6891a2ae0e8692679c07728819b6e2b822fb30ca7445f67bbf6509b25a96332c",
+ "sha256:6a750aec7bf431517a9fd78cb93c97b9b0c496090fee84a47a0d23668976b4b0",
+ "sha256:6f5c4d41b2771c730ea1c34e458e781b18cc668d194958e0112455fff4e402b2",
+ "sha256:77450e6d20016ec41f43ca4a6c63e9fdde03f0ae3fe90e7c27bdbeaece8b1ed4",
+ "sha256:81b5efb2f126454586d0f40c4d834010979cb80785173d1586df845a632e4e6d",
+ "sha256:823be1deb01793da05ecb0484d6c9e20baebb39bd42b5d72636ae9cf8350dbd2",
+ "sha256:834b5230b5dfc0c1ec37b2fda433b271cbbc0e507560b5d1588e2cc1148cf1ce",
+ "sha256:847a35c4d58721c5dc3dba599878ebbdfd96784f3fb8bb2c356e123bdcd73f34",
+ "sha256:86110d7e1907ab36691f80b33eb2da87d780f4739ae773e5fc83fb272f88825f",
+ "sha256:8951eee36c57cd128f779e641e21eb40bc5073eb28b2d23f33eb0ef14ffb3f5d",
+ "sha256:8a7164fe2005d03c64fd3b85649891cd4953a8de53107940bf272500ba8a788b",
+ "sha256:8b8bab4c97248095ae0c4455b5a1cd1cdd96e4e4769306ab19dda135ea4cdb07",
+ "sha256:90afc12421df2b1b4dcc975f814e21bc1754640d502a2fbcc6d41e77af5ec312",
+ "sha256:938cb21650855054dc54dfd9120a851c974f95450f00683399006aa6e8abb057",
+ "sha256:942ba11e7dfb66dc70f9ae66b33452f51ac7bb90676da39a7345e99ffb55402d",
+ "sha256:972658f4a72d02b8abfa2581d92d59f59897d2e9f7e708fdabe922f9087773af",
+ "sha256:97736815b9cc893b2b7f663628e63f436018b75f44854c8027040e05230eeddb",
+ "sha256:98906207f29bc2c459ff64fa007afd10a8c8ac080f7e4d5beff4c97086a3dabd",
+ "sha256:99457f184ad90235cfe8461c4d70ab7dd2680e28821c29eca00252ba90308c78",
+ "sha256:a0d829524aaefdebccb869eed855e2d04c21d2d7479b6cada7ace5448416597b",
+ "sha256:a2fdd81edd64342c85ac7cf2753ccae0b79bf2dfa063785503cb85a7d3593223",
+ "sha256:a55b5b16c839df1070bc113c1f7f94a0af4433fcfa1b41799ce7606e5c79ce0a",
+ "sha256:a642295cd0c8df1b86fc3dced1d067874c353a188dc8e0f744626d49e9aa51c4",
+ "sha256:ab86ce7c8f9bea87b9d12c7f0af71102acbf5ecbc66c17796cff45dae54ef9a5",
+ "sha256:abc267fa9837245cc28ea6929f19fa335f3dc330a35d2e45509b6566dc18be23",
+ "sha256:ae1d6df168efb88d7d522664693607b80b4080be6750c913eefb77e34c12c71a",
+ "sha256:b2ebef0e0b4454320274f5e83a41844c63438fdc874ea40a8b5b4ecb7693f1c4",
+ "sha256:b48ece5bde2e768197a2d0f6e925f9d7e3e826f0ad2271120f8144a9db18d5c8",
+ "sha256:b7cdf28938ac6b8b49ae5e92f2735056a7ba99c9b110a474473fd71185c1af5d",
+ "sha256:bb4462bd43c2460774914b8525f79b00f8f407c945d50881568f294c1d9b4443",
+ "sha256:bc4ff9805858bd54d1a20efff925ccd89c9d2e7cf4986144b30802bf78091c3e",
+ "sha256:c1322d7dd74713dcc157a2b7898a564ab091ca6c58302d5c7b4c07296e3fd00f",
+ "sha256:c67598100338d5d985db1b3d21f3619ef392e185e71b8d52bceacc4a7771ea7e",
+ "sha256:ca26a1e73c48cfc54c4a76ff78df3727b9d9f4ccc8dbee4ae3f73306a591676d",
+ "sha256:d323a01da91851a4f17bf592faf46149c9169d68430b3146dcba2bb5e5719abc",
+ "sha256:dc1803ac5c32ec324c5261c7209e8f8ce88e83254c4e1aebdc8b0a39f9ddb443",
+ "sha256:e00a3f196329e08e43d99b79b286d60ce46bed10f2280d25a1718399457e06be",
+ "sha256:e85637bc8fe81ddb73fda9e56bab24560bdddfa98aa64f87aaa4e4b6730c23d2",
+ "sha256:e858ac0a25074ba4bce653f9b5d0a85b7456eaddadc0ce82d3878c22489fa4ee",
+ "sha256:eae237477a873ab46e8dd748e515c72c0c804fb380fbe6c85533c7de51f23a8f",
+ "sha256:ebef0dd9bf9b812bf75bda96743f2a6c5734a02092ae7f721c048d156d5fabae",
+ "sha256:ec3beeada09ff865c344ff3bc2f427f5e6c26401cc6113d77e372c3fdac73864",
+ "sha256:f76d0ad001edd426b92233d45c746fd08f467d56100fd8f30e9ace4b005266e4",
+ "sha256:f85d05aa0918283cf29a30b547b4df2fbb56b45b135f9e35b6807cb28bc47951",
+ "sha256:f9899c94762343f2cc2fc64c13e7cae4c3cc65cdfc87dd810a31654c9b7358cc"
],
"markers": "python_version >= '3.8'",
- "version": "==2.18.2"
+ "version": "==2.18.4"
},
"pytest": {
"hashes": [
- "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233",
- "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"
+ "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343",
+ "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"
],
"index": "pypi",
- "version": "==8.2.0"
+ "version": "==8.2.2"
},
"python-dateutil": {
"hashes": [
@@ -778,11 +792,11 @@
},
"requests": {
"hashes": [
- "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
- "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
+ "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760",
+ "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"
],
- "markers": "python_version >= '3.7'",
- "version": "==2.31.0"
+ "markers": "python_version >= '3.8'",
+ "version": "==2.32.3"
},
"six": {
"hashes": [
@@ -824,11 +838,11 @@
},
"typing-extensions": {
"hashes": [
- "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0",
- "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"
+ "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
+ "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
],
"markers": "python_version >= '3.8'",
- "version": "==4.11.0"
+ "version": "==4.12.2"
},
"tzdata": {
"hashes": [
@@ -840,11 +854,11 @@
},
"urllib3": {
"hashes": [
- "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07",
- "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"
+ "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472",
+ "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"
],
- "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
- "version": "==1.26.18"
+ "markers": "python_version >= '3.10'",
+ "version": "==2.2.2"
},
"werkzeug": {
"hashes": [
@@ -1030,61 +1044,61 @@
"develop": {
"coverage": {
"hashes": [
- "sha256:0646599e9b139988b63704d704af8e8df7fa4cbc4a1f33df69d97f36cb0a38de",
- "sha256:0cdcbc320b14c3e5877ee79e649677cb7d89ef588852e9583e6b24c2e5072661",
- "sha256:0d0a0f5e06881ecedfe6f3dd2f56dcb057b6dbeb3327fd32d4b12854df36bf26",
- "sha256:1434e088b41594baa71188a17533083eabf5609e8e72f16ce8c186001e6b8c41",
- "sha256:16db7f26000a07efcf6aea00316f6ac57e7d9a96501e990a36f40c965ec7a95d",
- "sha256:1cc0fe9b0b3a8364093c53b0b4c0c2dd4bb23acbec4c9240b5f284095ccf7981",
- "sha256:1fc81d5878cd6274ce971e0a3a18a8803c3fe25457165314271cf78e3aae3aa2",
- "sha256:2ec92012fefebee89a6b9c79bc39051a6cb3891d562b9270ab10ecfdadbc0c34",
- "sha256:39afcd3d4339329c5f58de48a52f6e4e50f6578dd6099961cf22228feb25f38f",
- "sha256:4a7b0ceee8147444347da6a66be737c9d78f3353b0681715b668b72e79203e4a",
- "sha256:4a9ca3f2fae0088c3c71d743d85404cec8df9be818a005ea065495bedc33da35",
- "sha256:4bf0655ab60d754491004a5efd7f9cccefcc1081a74c9ef2da4735d6ee4a6223",
- "sha256:4cc37def103a2725bc672f84bd939a6fe4522310503207aae4d56351644682f1",
- "sha256:4fc84a37bfd98db31beae3c2748811a3fa72bf2007ff7902f68746d9757f3746",
- "sha256:5037f8fcc2a95b1f0e80585bd9d1ec31068a9bcb157d9750a172836e98bc7a90",
- "sha256:54de9ef3a9da981f7af93eafde4ede199e0846cd819eb27c88e2b712aae9708c",
- "sha256:556cf1a7cbc8028cb60e1ff0be806be2eded2daf8129b8811c63e2b9a6c43bca",
- "sha256:57e0204b5b745594e5bc14b9b50006da722827f0b8c776949f1135677e88d0b8",
- "sha256:5a5740d1fb60ddf268a3811bcd353de34eb56dc24e8f52a7f05ee513b2d4f596",
- "sha256:5c3721c2c9e4c4953a41a26c14f4cef64330392a6d2d675c8b1db3b645e31f0e",
- "sha256:5fa567e99765fe98f4e7d7394ce623e794d7cabb170f2ca2ac5a4174437e90dd",
- "sha256:5fd215c0c7d7aab005221608a3c2b46f58c0285a819565887ee0b718c052aa4e",
- "sha256:6175d1a0559986c6ee3f7fccfc4a90ecd12ba0a383dcc2da30c2b9918d67d8a3",
- "sha256:61c4bf1ba021817de12b813338c9be9f0ad5b1e781b9b340a6d29fc13e7c1b5e",
- "sha256:6537e7c10cc47c595828b8a8be04c72144725c383c4702703ff4e42e44577312",
- "sha256:68f962d9b72ce69ea8621f57551b2fa9c70509af757ee3b8105d4f51b92b41a7",
- "sha256:7352b9161b33fd0b643ccd1f21f3a3908daaddf414f1c6cb9d3a2fd618bf2572",
- "sha256:796a79f63eca8814ca3317a1ea443645c9ff0d18b188de470ed7ccd45ae79428",
- "sha256:79afb6197e2f7f60c4824dd4b2d4c2ec5801ceb6ba9ce5d2c3080e5660d51a4f",
- "sha256:7a588d39e0925f6a2bff87154752481273cdb1736270642aeb3635cb9b4cad07",
- "sha256:8748731ad392d736cc9ccac03c9845b13bb07d020a33423fa5b3a36521ac6e4e",
- "sha256:8fe7502616b67b234482c3ce276ff26f39ffe88adca2acf0261df4b8454668b4",
- "sha256:9314d5678dcc665330df5b69c1e726a0e49b27df0461c08ca12674bcc19ef136",
- "sha256:9735317685ba6ec7e3754798c8871c2f49aa5e687cc794a0b1d284b2389d1bd5",
- "sha256:9981706d300c18d8b220995ad22627647be11a4276721c10911e0e9fa44c83e8",
- "sha256:9e78295f4144f9dacfed4f92935fbe1780021247c2fabf73a819b17f0ccfff8d",
- "sha256:b016ea6b959d3b9556cb401c55a37547135a587db0115635a443b2ce8f1c7228",
- "sha256:b6cf3764c030e5338e7f61f95bd21147963cf6aa16e09d2f74f1fa52013c1206",
- "sha256:beccf7b8a10b09c4ae543582c1319c6df47d78fd732f854ac68d518ee1fb97fa",
- "sha256:c0884920835a033b78d1c73b6d3bbcda8161a900f38a488829a83982925f6c2e",
- "sha256:c3e757949f268364b96ca894b4c342b41dc6f8f8b66c37878aacef5930db61be",
- "sha256:ca498687ca46a62ae590253fba634a1fe9836bc56f626852fb2720f334c9e4e5",
- "sha256:d1d0d98d95dd18fe29dc66808e1accf59f037d5716f86a501fc0256455219668",
- "sha256:d21918e9ef11edf36764b93101e2ae8cc82aa5efdc7c5a4e9c6c35a48496d601",
- "sha256:d7fed867ee50edf1a0b4a11e8e5d0895150e572af1cd6d315d557758bfa9c057",
- "sha256:db66fc317a046556a96b453a58eced5024af4582a8dbdc0c23ca4dbc0d5b3146",
- "sha256:dde0070c40ea8bb3641e811c1cfbf18e265d024deff6de52c5950677a8fb1e0f",
- "sha256:df4e745a81c110e7446b1cc8131bf986157770fa405fe90e15e850aaf7619bc8",
- "sha256:e2213def81a50519d7cc56ed643c9e93e0247f5bbe0d1247d15fa520814a7cd7",
- "sha256:ef48e2707fb320c8f139424a596f5b69955a85b178f15af261bab871873bb987",
- "sha256:f152cbf5b88aaeb836127d920dd0f5e7edff5a66f10c079157306c4343d86c19",
- "sha256:fc0b4d8bfeabd25ea75e94632f5b6e047eef8adaed0c2161ada1e922e7f7cece"
+ "sha256:015eddc5ccd5364dcb902eaecf9515636806fa1e0d5bef5769d06d0f31b54523",
+ "sha256:04aefca5190d1dc7a53a4c1a5a7f8568811306d7a8ee231c42fb69215571944f",
+ "sha256:05ac5f60faa0c704c0f7e6a5cbfd6f02101ed05e0aee4d2822637a9e672c998d",
+ "sha256:0bbddc54bbacfc09b3edaec644d4ac90c08ee8ed4844b0f86227dcda2d428fcb",
+ "sha256:1d2a830ade66d3563bb61d1e3c77c8def97b30ed91e166c67d0632c018f380f0",
+ "sha256:239a4e75e09c2b12ea478d28815acf83334d32e722e7433471fbf641c606344c",
+ "sha256:244f509f126dc71369393ce5fea17c0592c40ee44e607b6d855e9c4ac57aac98",
+ "sha256:25a5caf742c6195e08002d3b6c2dd6947e50efc5fc2c2205f61ecb47592d2d83",
+ "sha256:296a7d9bbc598e8744c00f7a6cecf1da9b30ae9ad51c566291ff1314e6cbbed8",
+ "sha256:2e079c9ec772fedbade9d7ebc36202a1d9ef7291bc9b3a024ca395c4d52853d7",
+ "sha256:33ca90a0eb29225f195e30684ba4a6db05dbef03c2ccd50b9077714c48153cac",
+ "sha256:33fc65740267222fc02975c061eb7167185fef4cc8f2770267ee8bf7d6a42f84",
+ "sha256:341dd8f61c26337c37988345ca5c8ccabeff33093a26953a1ac72e7d0103c4fb",
+ "sha256:34d6d21d8795a97b14d503dcaf74226ae51eb1f2bd41015d3ef332a24d0a17b3",
+ "sha256:3538d8fb1ee9bdd2e2692b3b18c22bb1c19ffbefd06880f5ac496e42d7bb3884",
+ "sha256:38a3b98dae8a7c9057bd91fbf3415c05e700a5114c5f1b5b0ea5f8f429ba6614",
+ "sha256:3d5a67f0da401e105753d474369ab034c7bae51a4c31c77d94030d59e41df5bd",
+ "sha256:50084d3516aa263791198913a17354bd1dc627d3c1639209640b9cac3fef5807",
+ "sha256:55f689f846661e3f26efa535071775d0483388a1ccfab899df72924805e9e7cd",
+ "sha256:5bc5a8c87714b0c67cfeb4c7caa82b2d71e8864d1a46aa990b5588fa953673b8",
+ "sha256:62bda40da1e68898186f274f832ef3e759ce929da9a9fd9fcf265956de269dbc",
+ "sha256:705f3d7c2b098c40f5b81790a5fedb274113373d4d1a69e65f8b68b0cc26f6db",
+ "sha256:75e3f4e86804023e991096b29e147e635f5e2568f77883a1e6eed74512659ab0",
+ "sha256:7b2a19e13dfb5c8e145c7a6ea959485ee8e2204699903c88c7d25283584bfc08",
+ "sha256:7cec2af81f9e7569280822be68bd57e51b86d42e59ea30d10ebdbb22d2cb7232",
+ "sha256:8383a6c8cefba1b7cecc0149415046b6fc38836295bc4c84e820872eb5478b3d",
+ "sha256:8c836309931839cca658a78a888dab9676b5c988d0dd34ca247f5f3e679f4e7a",
+ "sha256:8e317953bb4c074c06c798a11dbdd2cf9979dbcaa8ccc0fa4701d80042d4ebf1",
+ "sha256:923b7b1c717bd0f0f92d862d1ff51d9b2b55dbbd133e05680204465f454bb286",
+ "sha256:990fb20b32990b2ce2c5f974c3e738c9358b2735bc05075d50a6f36721b8f303",
+ "sha256:9aad68c3f2566dfae84bf46295a79e79d904e1c21ccfc66de88cd446f8686341",
+ "sha256:a5812840d1d00eafae6585aba38021f90a705a25b8216ec7f66aebe5b619fb84",
+ "sha256:a6519d917abb15e12380406d721e37613e2a67d166f9fb7e5a8ce0375744cd45",
+ "sha256:ab0b028165eea880af12f66086694768f2c3139b2c31ad5e032c8edbafca6ffc",
+ "sha256:aea7da970f1feccf48be7335f8b2ca64baf9b589d79e05b9397a06696ce1a1ec",
+ "sha256:b1196e13c45e327d6cd0b6e471530a1882f1017eb83c6229fc613cd1a11b53cd",
+ "sha256:b368e1aee1b9b75757942d44d7598dcd22a9dbb126affcbba82d15917f0cc155",
+ "sha256:bde997cac85fcac227b27d4fb2c7608a2c5f6558469b0eb704c5726ae49e1c52",
+ "sha256:c4c2872b3c91f9baa836147ca33650dc5c172e9273c808c3c3199c75490e709d",
+ "sha256:c59d2ad092dc0551d9f79d9d44d005c945ba95832a6798f98f9216ede3d5f485",
+ "sha256:d1da0a2e3b37b745a2b2a678a4c796462cf753aebf94edcc87dcc6b8641eae31",
+ "sha256:d8b7339180d00de83e930358223c617cc343dd08e1aa5ec7b06c3a121aec4e1d",
+ "sha256:dd4b3355b01273a56b20c219e74e7549e14370b31a4ffe42706a8cda91f19f6d",
+ "sha256:e08c470c2eb01977d221fd87495b44867a56d4d594f43739a8028f8646a51e0d",
+ "sha256:f5102a92855d518b0996eb197772f5ac2a527c0ec617124ad5242a3af5e25f85",
+ "sha256:f542287b1489c7a860d43a7d8883e27ca62ab84ca53c965d11dac1d3a1fab7ce",
+ "sha256:f78300789a708ac1f17e134593f577407d52d0417305435b134805c4fb135adb",
+ "sha256:f81bc26d609bf0fbc622c7122ba6307993c83c795d2d6f6f6fd8c000a770d974",
+ "sha256:f836c174c3a7f639bded48ec913f348c4761cbf49de4a20a956d3431a7c9cb24",
+ "sha256:fa21a04112c59ad54f69d80e376f7f9d0f5f9123ab87ecd18fbb9ec3a2beed56",
+ "sha256:fcf7d1d6f5da887ca04302db8e0e0cf56ce9a5e05f202720e49b3e8157ddb9a9",
+ "sha256:fd27d8b49e574e50caa65196d908f80e4dff64d7e592d0c59788b45aad7e8b35"
],
"index": "pypi",
- "version": "==7.5.1"
+ "version": "==7.5.3"
},
"iniconfig": {
"hashes": [
@@ -1096,11 +1110,11 @@
},
"packaging": {
"hashes": [
- "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5",
- "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"
+ "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002",
+ "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
],
- "markers": "python_version >= '3.7'",
- "version": "==24.0"
+ "markers": "python_version >= '3.8'",
+ "version": "==24.1"
},
"pluggy": {
"hashes": [
@@ -1112,11 +1126,11 @@
},
"pytest": {
"hashes": [
- "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233",
- "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"
+ "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343",
+ "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"
],
"index": "pypi",
- "version": "==8.2.0"
+ "version": "==8.2.2"
}
}
}
diff --git a/dbrepo-search-service/init/app.py b/dbrepo-search-service/init/app.py
index cf2fb12b33aa5ffec44887f4b87de41723b2472a..82617fccb44d93ae2cb741a5b31e89dce26620d8 100644
--- a/dbrepo-search-service/init/app.py
+++ b/dbrepo-search-service/init/app.py
@@ -102,27 +102,29 @@ class App:
def fetch_databases(self) -> List[Database]:
client = RestClient(endpoint=self.gateway_endpoint)
- databases = client.get_databases()
+ databases = []
+ for database in client.get_databases():
+ databases.append(client.get_database(database_id=database.id))
logging.debug(f"fetched {len(databases)} database(s)")
return databases
- def save_databases(self, databases: List[Database], index_created: bool, index_updated: bool):
+ def save_databases(self, databases: List[Database]):
logging.debug(
- f"save {len(databases)} database(s), index_created={index_created}, index_updated={index_updated}")
+ f"save {len(databases)} database(s)")
for doc in databases:
doc: Database = doc
- if index_created:
- self._instance().create(index="database", id=doc.id, body=doc.model_dump())
- logging.info(f"Saved database with id {doc.id}")
- elif index_updated:
+ try:
self._instance().delete(index="database", id=doc.id)
- self._instance().create(index="database", id=doc.id, body=doc.model_dump())
- logging.info(f"Updated database with id {doc.id}")
+ logging.debug(f"deleted database with id {doc.id}")
+ except opensearchpy.NotFoundError:
+ logging.warning(f"Database with id {doc.id} does not exist, skip.")
+ self._instance().create(index="database", id=doc.id, body=doc.model_dump())
+ logging.debug(f"created database with id {doc.id}")
if __name__ == "__main__":
app = App()
create = not app.index_exists()
update = app.index_update(is_created=create)
- app.save_databases(databases=app.fetch_databases(), index_created=create, index_updated=update)
+ app.save_databases(databases=app.fetch_databases())
logging.info("Finished. Exiting.")
diff --git a/dbrepo-search-service/init/lib/dbrepo-1.4.4-py3-none-any.whl b/dbrepo-search-service/init/lib/dbrepo-1.4.4-py3-none-any.whl
new file mode 100644
index 0000000000000000000000000000000000000000..617969c3eb15926d932b7c0180bed51b9ef7052d
Binary files /dev/null and b/dbrepo-search-service/init/lib/dbrepo-1.4.4-py3-none-any.whl differ
diff --git a/dbrepo-search-service/init/lib/dbrepo-1.4.4.tar.gz b/dbrepo-search-service/init/lib/dbrepo-1.4.4.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..9d1d5ae238baba6bc51db4d219a0d09b5aca1c51
Binary files /dev/null and b/dbrepo-search-service/init/lib/dbrepo-1.4.4.tar.gz differ
diff --git a/dbrepo-search-service/lib/dbrepo-1.4.4-py3-none-any.whl b/dbrepo-search-service/lib/dbrepo-1.4.4-py3-none-any.whl
index 7e8fd7fca5aa6158bf57952f7f1050a08b331402..617969c3eb15926d932b7c0180bed51b9ef7052d 100644
Binary files a/dbrepo-search-service/lib/dbrepo-1.4.4-py3-none-any.whl and b/dbrepo-search-service/lib/dbrepo-1.4.4-py3-none-any.whl differ
diff --git a/dbrepo-search-service/lib/dbrepo-1.4.4.tar.gz b/dbrepo-search-service/lib/dbrepo-1.4.4.tar.gz
index 3e45d4513a31a1fda334ed9ad2c5cbad3803199a..9d1d5ae238baba6bc51db4d219a0d09b5aca1c51 100644
Binary files a/dbrepo-search-service/lib/dbrepo-1.4.4.tar.gz and b/dbrepo-search-service/lib/dbrepo-1.4.4.tar.gz differ
diff --git a/lib/python/Makefile b/lib/python/Makefile
index 7c79a178bfd875327542c07e6bbdd0353a068b19..229fa04db39df9ca8d20263cf3777b8b79e1febe 100644
--- a/lib/python/Makefile
+++ b/lib/python/Makefile
@@ -19,8 +19,8 @@ install:
(cd ../../dbrepo-analyse-service && pipenv lock)
cp ./dist/dbrepo-* ../../dbrepo-search-service/lib/
(cd ../../dbrepo-search-service && pipenv lock)
- cp ./dist/dbrepo-* ../../../dpm2024/lib/
- (cd ../../../dpm2024 && pipenv lock)
+ cp ./dist/dbrepo-* ../../dbrepo-search-service/init/lib/
+ (cd ../../dbrepo-search-service/init && pipenv lock)
deploy: build
python3 -m twine upload --config-file ~/.pypirc --verbose --repository pypi ./dist/dbrepo-*
diff --git a/lib/python/README.md b/lib/python/README.md
index c8785a2e84153752e2e3020b6dfb56c3a119ce04..35fec904439b74e2d1f91d04a19f15615a0ca372 100644
--- a/lib/python/README.md
+++ b/lib/python/README.md
@@ -15,72 +15,50 @@ This package supports Python 3.11+.
## Quickstart
-Create a table and import a .csv file from your computer.
+Get public data from a table as pandas `DataFrame`:
```python
from dbrepo.RestClient import RestClient
-from dbrepo.api.dto import CreateTableColumn, ColumnType, CreateTableConstraints
-client = RestClient(endpoint='https://test.dbrepo.tuwien.ac.at', username="foo",
- password="bar")
+client = RestClient(endpoint="https://dbrepo1.ec.tuwien.ac.at")
+# Get a small data slice of just three rows
+df = client.get_table_data(database_id=7, table_id=13, page=0, size=3, df=True)
+print(df)
+# x_coord component unit ... value stationid meantype
+# 0 16.52617 Feinstaub (PM10) µg/m³ ... 21.0 01:0001 HMW
+# 1 16.52617 Feinstaub (PM10) µg/m³ ... 23.0 01:0001 HMW
+# 2 16.52617 Feinstaub (PM10) µg/m³ ... 26.0 01:0001 HMW
+#
+# [3 rows x 12 columns]
+```
+
+Import data into a table:
+
+```python
+import pandas as pd
+from dbrepo.RestClient import RestClient
-# analyse csv
-analysis = client.analyse_datatypes(file_path="sensor.csv", separator=",")
-print(f"Analysis result: {analysis}")
-# -> columns=(date=date, precipitation=decimal, lat=decimal, lng=decimal), separator=,
-# line_termination=\n
-
-# create table
-table = client.create_table(database_id=1,
- name="Sensor Data",
- constraints=CreateTableConstraints(
- checks=['precipitation >= 0'],
- uniques=[['precipitation']]),
- columns=[CreateTableColumn(name="date",
- type=ColumnType.DATE,
- dfid=3, # YYYY-MM-dd
- primary_key=True,
- null_allowed=False),
- CreateTableColumn(name="precipitation",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True),
- CreateTableColumn(name="lat",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True),
- CreateTableColumn(name="lng",
- type=ColumnType.DECIMAL,
- size=10,
- d=4,
- primary_key=False,
- null_allowed=True)])
-print(f"Create table result {table}")
-# -> (id=1, internal_name=sensor_data, ...)
-
-client.import_table_data(database_id=1, table_id=1, file_path="sensor.csv", separator=",",
- skip_lines=1, line_encoding="\n")
-print(f"Finished.")
+client = RestClient(endpoint="https://dbrepo1.ec.tuwien.ac.at", username="foo",
+ password="bar")
+df = pd.DataFrame(data={'x_coord': 16.52617, 'component': 'Feinstaub (PM10)',
+ 'unit': 'µg/m³', ...})
+client.import_table_data(database_id=7, table_id=13, file_name_or_data_frame=df)
```
## Supported Features & Best-Practices
- Manage user
- account ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//usage-overview/#create-user-account))
+ account ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#create-user-account))
- Manage
databases ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//usage-overview/#create-database))
- Manage database access &
- visibility ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//usage-overview/#private-database-access))
+ visibility ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#create-database))
- Import
- dataset ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//usage-overview/#private-database-access))
+ dataset ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#import-dataset))
- Create persistent
- identifiers ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//usage-overview/#assign-database-pid))
+ identifiers ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#assign-database-pid))
- Execute
- queries ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo//usage-overview/#export-subset))
+ queries ([docs](https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.4.4/api/#export-subset))
- Get data from tables/views/subsets
## Configure
@@ -89,16 +67,26 @@ All credentials can optionally be set/overridden with environment variables. Thi
Jupyter Notebooks by creating an invisible `.env` file and loading it:
```
-REST_API_ENDPOINT="https://test.dbrepo.tuwien.ac.at"
+REST_API_ENDPOINT="https://dbrepo1.ec.tuwien.ac.at"
REST_API_USERNAME="foo"
REST_API_PASSWORD="bar"
REST_API_SECURE="True"
-AMQP_API_HOST="https://test.dbrepo.tuwien.ac.at"
+AMQP_API_HOST="https://dbrepo1.ec.tuwien.ac.at"
AMQP_API_PORT="5672"
AMQP_API_USERNAME="foo"
AMQP_API_PASSWORD="bar"
-AMQP_API_VIRTUAL_HOST="/"
-REST_UPLOAD_ENDPOINT="https://test.dbrepo.tuwien.ac.at/api/upload/files"
+AMQP_API_VIRTUAL_HOST="dbrepo"
+REST_UPLOAD_ENDPOINT="https://dbrepo1.ec.tuwien.ac.at/api/upload/files"
+```
+
+You can disable logging by setting the log level to e.g. `INFO`:
+
+```python
+from dbrepo.RestClient import RestClient
+import logging
+logging.getLogger().setLevel(logging.INFO)
+...
+client = RestClient(...)
```
## Roadmap
diff --git a/lib/python/dbrepo/AmqpClient.py b/lib/python/dbrepo/AmqpClient.py
index 1afcc2a7b582e7fdac634d6d7a14fdd35c4d3797..f570c5d4cc103dda211e1a74839687b9a3f1a6a3 100644
--- a/lib/python/dbrepo/AmqpClient.py
+++ b/lib/python/dbrepo/AmqpClient.py
@@ -7,6 +7,8 @@ import logging
from dbrepo.api.dto import CreateData
+logger = logging.getLogger("AmqpClient")
+
class AmqpClient:
"""
@@ -58,5 +60,5 @@ class AmqpClient:
connection = pika.BlockingConnection(parameters)
channel = connection.channel()
channel.basic_publish(exchange=exchange, routing_key=routing_key,
- body=json.dumps(dataclasses.asdict(CreateData(data=data))))
+ body=CreateData(data=data).model_dump_json())
connection.close()
diff --git a/lib/python/dbrepo/RestClient.py b/lib/python/dbrepo/RestClient.py
index ae956dd07277830235c6d3ca991190766f8fd7de..a2687862da671ffdc2ab451299bfb3faa4d92097 100644
--- a/lib/python/dbrepo/RestClient.py
+++ b/lib/python/dbrepo/RestClient.py
@@ -1,6 +1,7 @@
-import sys
import os
+import sys
import logging
+import time
import requests
from pydantic import TypeAdapter
@@ -10,9 +11,11 @@ from pandas import DataFrame
from dbrepo.UploadClient import UploadClient
from dbrepo.api.dto import *
from dbrepo.api.exceptions import ResponseCodeError, UsernameExistsError, EmailExistsError, NotExistsError, \
- ForbiddenError, MalformedError, NameExistsError, QueryStoreError, MetadataConsistencyError, ExternalSystemError, \
+ ForbiddenError, MalformedError, NameExistsError, QueryStoreError, ExternalSystemError, \
AuthenticationError, UploadError, FormatNotAvailable, RequestError, ServiceError, ServiceConnectionError
+logger = logging.getLogger("RestClient")
+
class RestClient:
"""
@@ -45,7 +48,7 @@ class RestClient:
self.secure = os.environ.get('REST_API_SECURE') == 'True'
else:
self.secure = secure
- logging.debug(
+ logger.debug(
f'initialized rest client with endpoint={self.endpoint}, username={username}, verify_ssl={secure}')
def _wrapper(self, method: str, url: str, params: [(str,)] = None, payload=None, headers: dict = None,
@@ -53,27 +56,27 @@ class RestClient:
if force_auth and (self.username is None and self.password is None):
raise AuthenticationError(f"Failed to perform request: authentication required")
url = f'{self.endpoint}{url}'
- logging.debug(f'method: {method}')
- logging.debug(f'url: {url}')
+ logger.debug(f'method: {method}')
+ logger.debug(f'url: {url}')
if params is not None:
- logging.debug(f'params: {params}')
+ logger.debug(f'params: {params}')
if stream is not None:
- logging.debug(f'stream: {stream}')
- logging.debug(f'secure: {self.secure}')
+ logger.debug(f'stream: {stream}')
+ logger.debug(f'secure: {self.secure}')
if headers is not None:
- logging.debug(f'headers: {headers}')
+ logger.debug(f'headers: {headers}')
else:
headers = dict()
- logging.debug(f'no headers set')
+ logger.debug(f'no headers set')
if payload is not None:
payload = payload.model_dump()
auth = None
if self.username is None and self.password is not None:
headers["Authorization"] = f"Bearer {self.password}"
- logging.debug(f'configured for oidc/bearer auth')
+ logger.debug(f'configured for oidc/bearer auth')
elif self.username is not None and self.password is not None:
auth = (self.username, self.password)
- logging.debug(f'configured for basic auth: username={self.username}, password=(hidden)')
+ logger.debug(f'configured for basic auth: username={self.username}, password=(hidden)')
return requests.request(method=method, url=url, auth=auth, verify=self.secure,
json=payload, headers=headers, params=params, stream=stream)
@@ -165,9 +168,9 @@ class RestClient:
:returns: The username, if set.
"""
if self.username is not None:
- logging.info(f"{self.username}")
+ print(f"{self.username}")
return self.username
- logging.info(f"No username set!")
+ print(f"No username set!")
return None
def get_users(self) -> List[UserBrief]:
@@ -369,7 +372,7 @@ class RestClient:
raise ResponseCodeError(f'Failed to get container: response code: {response.status_code} is not '
f'200 (OK): {response.text}')
- def get_databases(self) -> List[Database]:
+ def get_databases(self) -> List[DatabaseBrief]:
"""
Get all databases.
@@ -381,7 +384,7 @@ class RestClient:
response = self._wrapper(method="get", url=url)
if response.status_code == 200:
body = response.json()
- return TypeAdapter(List[Database]).validate_python(body)
+ return TypeAdapter(List[DatabaseBrief]).validate_python(body)
raise ResponseCodeError(f'Failed to find databases: response code: {response.status_code} is not '
f'200 (OK): {response.text}')
@@ -1096,9 +1099,9 @@ class RestClient:
raise ResponseCodeError(f'Failed to insert table data: response code: {response.status_code} is not '
f'201 (CREATED): {response.text}')
- def import_table_data(self, database_id: int, table_id: int, separator: str, file_path: str,
- quote: str = None, skip_lines: int = 0, false_encoding: str = None,
- true_encoding: str = None, null_encoding: str = None,
+ def import_table_data(self, database_id: int, table_id: int, separator: str,
+ file_name_or_data_frame: str | DataFrame, quote: str = None, skip_lines: int = 0,
+ false_encoding: str = None, true_encoding: str = None, null_encoding: str = None,
line_encoding: str = "\r\n") -> None:
"""
Import a csv dataset from a file into a table in a database with given database id and table id.
@@ -1106,7 +1109,7 @@ class RestClient:
:param database_id: The database id.
:param table_id: The table id.
:param separator: The csv column separator.
- :param file_path: The path of the file that is imported on the storage service.
+ :param file_name_or_data_frame: The path of the file that is imported on the storage service or pandas dataframe.
:param quote: The column data quotation character. Optional.
:param skip_lines: The number of lines to skip. Optional. Default: 0.
:param false_encoding: The encoding of boolean false. Optional.
@@ -1121,6 +1124,12 @@ class RestClient:
:raises ResponseCodeError: If something went wrong with the insert.
"""
client = UploadClient(endpoint=f"{self.endpoint}/api/upload/files")
+ if type(file_name_or_data_frame) is DataFrame:
+ file_path: str = f"./tmp-{time.time()}"
+ df: DataFrame = file_name_or_data_frame
+ df.to_csv(path_or_buf=file_path, index=False, header=False)
+ else:
+ file_path: str = file_name_or_data_frame
filename = client.upload(file_path=file_path)
url = f'/api/database/{database_id}/table/{table_id}/data/import'
response = self._wrapper(method="post", url=url, force_auth=True,
@@ -1543,7 +1552,8 @@ class RestClient:
raise ResponseCodeError(f'Failed to delete database access: response code: {response.status_code} is not '
f'201 (CREATED): {response.text}')
- def create_subset(self, database_id: int, query: str, page: int = 0, size: int = 10) -> Result:
+ def create_subset(self, database_id: int, query: str, page: int = 0, size: int = 10,
+ df: bool = False) -> Result | DataFrame:
"""
Executes a SQL query in a database where the current user has at least read access with given database id. The
result set can be paginated with setting page and size (both). Historic data can be queried by setting
@@ -1553,6 +1563,7 @@ class RestClient:
:param query: The query statement.
:param page: The result pagination number. Optional. Default: 0.
:param size: The result pagination size. Optional. Default: 10.
+ :param df: If true, the result is returned as Pandas DataFrame. Optional. Default: False.
:returns: The result set, if successful.
@@ -1571,7 +1582,10 @@ class RestClient:
payload=ExecuteQuery(statement=query))
if response.status_code == 201:
body = response.json()
- return Result.model_validate(body)
+ res = Result.model_validate(body)
+ if df:
+ return DataFrame.from_records(res.result)
+ return res
if response.status_code == 400:
raise MalformedError(f'Failed to create subset: {response.text}')
if response.status_code == 403:
diff --git a/lib/python/dbrepo/UploadClient.py b/lib/python/dbrepo/UploadClient.py
index ebcb5aba57bad1eba1c3e7040bc284f702465c83..05fa133d64b2be902deb81423fef9bdc07e8c66d 100644
--- a/lib/python/dbrepo/UploadClient.py
+++ b/lib/python/dbrepo/UploadClient.py
@@ -4,6 +4,8 @@ import re
import sys
from tusclient import client
+logger = logging.getLogger("UploadClient")
+
class UploadClient:
"""
@@ -33,5 +35,5 @@ class UploadClient:
uploader.upload()
m = re.search('\\/([a-f0-9]+)\\+', uploader.url)
filename = m.group(0)[1:-1]
- logging.debug(f'uploaded file {file_path} to storage service with key: {filename}')
+ logger.debug(f'uploaded file {file_path} to storage service with key: {filename}')
return filename
diff --git a/lib/python/dbrepo/api/dto.py b/lib/python/dbrepo/api/dto.py
index 20d7a8252354bd6b6523eb2026feeff27ed71967..c601125767c2065e62062c9c82e39ad402ee5bc8 100644
--- a/lib/python/dbrepo/api/dto.py
+++ b/lib/python/dbrepo/api/dto.py
@@ -993,6 +993,18 @@ class Database(BaseModel):
exchange_type: Optional[str] = None
+class DatabaseBrief(BaseModel):
+ id: int
+ name: str
+ internal_name: str
+ description: Optional[str] = None
+ is_public: bool
+ identifiers: Optional[List[Identifier]] = field(default_factory=list)
+ contact: UserBrief
+ owner: UserBrief
+ created: Timestamp
+
+
class Unique(BaseModel):
id: int
table: TableMinimal
diff --git a/lib/python/tests/test_unit_database.py b/lib/python/tests/test_unit_database.py
index cb0bb19a702233cc3b573c028bd4fe1589e54fa3..dea15691e22a990434772c433683eb6eefc0b253 100644
--- a/lib/python/tests/test_unit_database.py
+++ b/lib/python/tests/test_unit_database.py
@@ -6,7 +6,8 @@ import datetime
from pydantic_core import ValidationError
from dbrepo.RestClient import RestClient
-from dbrepo.api.dto import Database, User, Container, Image, UserAttributes, DatabaseAccess, AccessType
+from dbrepo.api.dto import Database, User, Container, Image, UserAttributes, DatabaseAccess, AccessType, DatabaseBrief, \
+ UserBrief
from dbrepo.api.exceptions import ResponseCodeError, NotExistsError, ForbiddenError, MalformedError, AuthenticationError
from dbrepo.api.dto import ImageDate
@@ -24,47 +25,14 @@ class DatabaseUnitTest(unittest.TestCase):
def test_get_databases_succeeds(self):
exp = [
- Database(
+ DatabaseBrief(
id=1,
name='test',
- creator=User(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise',
- attributes=UserAttributes(theme='light')),
- owner=User(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise',
- attributes=UserAttributes(theme='light')),
- contact=User(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise',
- attributes=UserAttributes(theme='light')),
+ owner=UserBrief(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise'),
+ contact=UserBrief(id='8638c043-5145-4be8-a3e4-4b79991b0a16', username='mweise'),
created=datetime.datetime(2024, 1, 1, 0, 0, 0, 0, datetime.timezone.utc),
- exchange_name='dbrepo',
internal_name='test_abcd',
- is_public=True,
- container=Container(
- id=1,
- name='MariaDB Galera 11.1.3',
- internal_name='mariadb',
- host='data-db',
- port=3306,
- sidecar_host='data-db-sidecar',
- sidecar_port=3305,
- created=datetime.datetime(2024, 1, 1, 0, 0, 0, 0, datetime.timezone.utc),
- image=Image(
- id=1,
- registry='docker.io',
- name='mariadb',
- version='11.2.2',
- dialect='org.hibernate.dialect.MariaDBDialect',
- driver_class='org.mariadb.jdbc.Driver',
- jdbc_method='mariadb',
- default_port=3306,
- date_formats=[ImageDate(id=1,
- created_at=datetime.datetime(2024, 3, 25, 18, 2, 14, 0,
- datetime.timezone.utc),
- example="2024-03-25 18:02:14",
- database_format='%Y-%c-%d %H:%i:%S',
- unix_format='yyyy-MM-dd HH:mm:ss',
- has_time=True)]
- )
- )
- )
+ is_public=True)
]
with requests_mock.Mocker() as mock:
# mock
diff --git a/mkdocs.yml b/mkdocs.yml
index 0c9c1d2a878b47e304fbcde67a78fcf89d98d900..8939c5fa446d9921fe20b21f9ca51d9713287e2d 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -45,10 +45,14 @@ nav:
- UI:
- Customization: api/ui.md
- Examples:
- - Hazardous Materials: examples/hazard.md
- - Power in Industry 4.0: examples/power.md
- - Transportation Monitoring: examples/transportation.md
- - XPS: examples/xps.md
+ - Air Quality Data: examples/air.md
+ - COVID-19 Data: examples/covid-19.md
+ - Hazard Data: examples/hazard.md
+ - Industry 4.0 Power Data: examples/power.md
+ - Survey Data: examples/survey.md
+ - Music-ML Data: examples/music.md
+ - Transportation Data: examples/transportation.md
+ - XPS Data: examples/xps-data.md
- publications.md
- contact.md
extra_css: