Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • fair-data-austria-db-repository/fda-services
1 result
Select Git revision
Show changes
Commits on Source (12)
Showing
with 6430 additions and 6205 deletions
...@@ -3,7 +3,6 @@ volumes: ...@@ -3,7 +3,6 @@ volumes:
data-db-data: data-db-data:
auth-db-data: auth-db-data:
broker-service-data: broker-service-data:
upload-service-data:
search-db-data: search-db-data:
identity-service-data: identity-service-data:
metric-db-data: metric-db-data:
...@@ -114,7 +113,7 @@ services: ...@@ -114,7 +113,7 @@ services:
init: true init: true
restart: "no" restart: "no"
container_name: dbrepo-auth-service-init container_name: dbrepo-auth-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.6.5 image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.7.0
environment: environment:
AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-admin} AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-admin}
AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-admin} AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-admin}
...@@ -135,7 +134,7 @@ services: ...@@ -135,7 +134,7 @@ services:
restart: "no" restart: "no"
container_name: dbrepo-metadata-service container_name: dbrepo-metadata-service
hostname: metadata-service hostname: metadata-service
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.6.5 image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.7.0
volumes: volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp" - "${SHARED_VOLUME:-/tmp}:/tmp"
environment: environment:
...@@ -198,7 +197,7 @@ services: ...@@ -198,7 +197,7 @@ services:
restart: "no" restart: "no"
container_name: dbrepo-analyse-service container_name: dbrepo-analyse-service
hostname: analyse-service hostname: analyse-service
image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.6.5 image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.7.0
environment: environment:
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client} AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG} AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
...@@ -253,7 +252,7 @@ services: ...@@ -253,7 +252,7 @@ services:
restart: "no" restart: "no"
container_name: dbrepo-search-db container_name: dbrepo-search-db
hostname: search-db hostname: search-db
image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.6.5 image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.7.0
healthcheck: healthcheck:
test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP
interval: 10s interval: 10s
...@@ -277,7 +276,7 @@ services: ...@@ -277,7 +276,7 @@ services:
restart: "no" restart: "no"
container_name: dbrepo-search-service container_name: dbrepo-search-service
hostname: search-service hostname: search-service
image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.6.5 image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.7.0
environment: environment:
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client} AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT_SECRET:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG} AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT_SECRET:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
...@@ -301,11 +300,10 @@ services: ...@@ -301,11 +300,10 @@ services:
restart: "no" restart: "no"
container_name: dbrepo-ui container_name: dbrepo-ui
hostname: ui hostname: ui
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.6.5 image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.7.0
environment: environment:
NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}" NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}"
NUXT_PUBLIC_API_SERVER: "${BASE_URL:-http://gateway-service}" NUXT_PUBLIC_API_SERVER: "${BASE_URL:-http://gateway-service}"
NUXT_PUBLIC_UPLOAD_CLIENT: "${BASE_URL:-http://localhost}/api/upload/files"
NUXT_OIDC_PROVIDERS_KEYCLOAK_AUTHORIZATION_URL: "${BASE_URL:-http://localhost}/realms/dbrepo/protocol/openid-connect/auth" NUXT_OIDC_PROVIDERS_KEYCLOAK_AUTHORIZATION_URL: "${BASE_URL:-http://localhost}/realms/dbrepo/protocol/openid-connect/auth"
NUXT_OIDC_PROVIDERS_KEYCLOAK_BASE_URL: "${BASE_URL:-http://localhost}/realms/dbrepo" NUXT_OIDC_PROVIDERS_KEYCLOAK_BASE_URL: "${BASE_URL:-http://localhost}/realms/dbrepo"
NUXT_OIDC_PROVIDERS_KEYCLOAK_CLIENT_ID: "${AUTH_SERVICE_CLIENT:-dbrepo-client}" NUXT_OIDC_PROVIDERS_KEYCLOAK_CLIENT_ID: "${AUTH_SERVICE_CLIENT:-dbrepo-client}"
...@@ -318,8 +316,6 @@ services: ...@@ -318,8 +316,6 @@ services:
depends_on: depends_on:
dbrepo-search-service: dbrepo-search-service:
condition: service_healthy condition: service_healthy
dbrepo-upload-service:
condition: service_healthy
healthcheck: healthcheck:
test: curl -fsSL http://127.0.0.1:3000 && curl -fsSL http://127.0.0.1:3000/health test: curl -fsSL http://127.0.0.1:3000 && curl -fsSL http://127.0.0.1:3000/health
interval: 10s interval: 10s
...@@ -381,7 +377,7 @@ services: ...@@ -381,7 +377,7 @@ services:
init: true init: true
container_name: dbrepo-search-service-init container_name: dbrepo-search-service-init
hostname: search-service-init hostname: search-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.6.5 image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.7.0
environment: environment:
LOG_LEVEL: ${LOG_LEVEL:-info} LOG_LEVEL: ${LOG_LEVEL:-info}
METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080} METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
...@@ -438,7 +434,7 @@ services: ...@@ -438,7 +434,7 @@ services:
restart: "no" restart: "no"
container_name: dbrepo-dashboard-service container_name: dbrepo-dashboard-service
hostname: dashboard-service hostname: dashboard-service
image: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.6.5 image: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.7.0
ports: ports:
- "3000:3000" - "3000:3000"
volumes: volumes:
...@@ -465,7 +461,7 @@ services: ...@@ -465,7 +461,7 @@ services:
init: true init: true
container_name: dbrepo-storage-service-init container_name: dbrepo-storage-service-init
hostname: storage-service-init hostname: storage-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.6.5 image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.7.0
environment: environment:
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-seaweedfsadmin} S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-seaweedfsadmin}
S3_BUCKET: "${S3_BUCKET:-dbrepo}" S3_BUCKET: "${S3_BUCKET:-dbrepo}"
...@@ -477,41 +473,11 @@ services: ...@@ -477,41 +473,11 @@ services:
logging: logging:
driver: json-file driver: json-file
dbrepo-upload-service:
restart: "no"
container_name: dbrepo-upload-service
hostname: upload-service
image: docker.io/tusproject/tusd:v2.4.0
volumes:
- "./config/pre-create.sh:/srv/tusd-hooks/pre-create:ro"
command:
- "-behind-proxy"
- "-max-size=2000000000"
- "-base-path=/api/upload/files/"
- "-hooks-dir=/srv/tusd-hooks/"
- "-s3-endpoint=${STORAGE_ENDPOINT:-http://storage-service:9000}"
- "-s3-bucket=dbrepo"
environment:
AWS_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
AWS_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
AWS_REGION: "${STORAGE_REGION_NAME:-default}"
METADATA_SERVICE_ENDPOINT: "${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}"
depends_on:
dbrepo-storage-service:
condition: service_healthy
healthcheck:
test: wget -qO- localhost:8080/metrics | grep "tusd" || exit 1
interval: 10s
timeout: 5s
retries: 12
logging:
driver: json-file
dbrepo-data-service: dbrepo-data-service:
restart: "no" restart: "no"
container_name: dbrepo-data-service container_name: dbrepo-data-service
hostname: data-service hostname: data-service
image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.6.5 image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.7.0
volumes: volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp" - "${SHARED_VOLUME:-/tmp}:/tmp"
environment: environment:
......
This diff is collapsed.
This diff is collapsed.
...@@ -11,7 +11,7 @@ components: ...@@ -11,7 +11,7 @@ components:
type: http type: http
externalDocs: externalDocs:
description: Project Website description: Project Website
url: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.6/ url: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.7/
info: info:
contact: contact:
email: andreas.rauber@tuwien.ac.at email: andreas.rauber@tuwien.ac.at
...@@ -24,7 +24,7 @@ info: ...@@ -24,7 +24,7 @@ info:
name: Apache 2.0 name: Apache 2.0
url: https://www.apache.org/licenses/LICENSE-2.0 url: https://www.apache.org/licenses/LICENSE-2.0
title: DBRepo REST API title: DBRepo REST API
version: 1.6.5 version: 1.7.0
openapi: 3.1.0 openapi: 3.1.0
servers: servers:
- description: Test Instance - description: Test Instance
......
This diff is collapsed.
...@@ -22,19 +22,23 @@ author: Martin Weise ...@@ -22,19 +22,23 @@ author: Martin Weise
We use [SeaweedFS](https://seaweedfs.github.io/) as a high-performance, S3 compatible object store for easy, cloud-ready We use [SeaweedFS](https://seaweedfs.github.io/) as a high-performance, S3 compatible object store for easy, cloud-ready
deployments that by default support replication and monitoring. No graphical user interface is provided out-of-the-box, deployments that by default support replication and monitoring. No graphical user interface is provided out-of-the-box,
administrators can access the S3 storage via S3-compatible clients administrators can access the S3 storage via S3-compatible clients
e.g. [AWS CLI](https://docs.aws.amazon.com/cli/latest/reference/s3/) (see below). e.g. [AWS CLI](https://docs.aws.amazon.com/cli/latest/reference/s3/) (see below).
### Users The default configuration creates admin credentials `seaweedfsadmin:seaweedfsadmin`. By default, one bucket `dbrepo` is
created that holds uploads temporarily. It is recommended to delete the contents regularly.
The default configuration creates one user `seaweedfsadmin` with password `seaweedfsadmin`. The S3 endpoint of the Storage Service is available on port `9000`.
### Buckets ### Filer UI
The default configuration creates two buckets `dbrepo-upload`, `dbrepo-download`: The storage service comes with a simple UI that can be used to explore the uploaded files, rename them and delete them.
Please note that the Filer UI is not intended for production and should be turned off for security purposes.
* `dbrepo-upload` for CSV-file upload (for import of data, analysis, etc.) from the User Interface <figure markdown>
* `dbrepo-download` for CSV-file download (exporting data, metadata, etc.) ![Filer UI with a list of uploaded files in the bucket dbrepo](../images/screenshots/storage-service-filer.png)
<figcaption>Figure 1: Filer UI</figcaption>
</figure>
## Limitations ## Limitations
......
---
author: Martin Weise
---
## tl;dr
!!! debug "Debug Information"
Image: [`docker.io/tusproject/tusd:v1.12`](https://hub.docker.com/r/tusproject/tusd)
* Ports: 1080/tcp
* Prometheus: `http://<hostname>:1080/api/upload/metrics`
* API: `http://<hostname>:1080/api/upload`
To directly access in Kubernetes (for e.g. debugging), forward the svc port to your local machine:
```shell
kubectl [-n namespace] port-forward svc/upload-service 1080:80
```
## Overview
We use the [TUS](https://tus.io/) open protocol for resume-able file uploads which based entirely on HTTP. Even though
the Upload Service is part of the standard installation, it is an entirely optional component and can be replaced with
any S3-compatible Blob Storage.
### Architecture
The Upload Service communicates internally with the [Storage Service](../storage-service) (c.f. [Figure 1](#fig1)).
<figure id="fig1" markdown>
![Architecture of the Upload Service](../images/architecture-upload-service.svg)
<figcaption>Figure 1: Architecture of the Upload Service</figcaption>
</figure>
The Upload Service is responsible for uploading files (mainly CSV-files) into a Blob Storage that can be accesses trough
the S3 protocol (e.g. our [Storage Service](../storage-service)). Make sure that the Upload Service can be
accessed from the Gateway Service.
## Limitations
* No support for authentication.
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
## Security
1. We strongly encourage to limit the clients allowed to upload by adding your subnet, e.g. `128.130.0.0/16`
(=TU Wien subnet) to the [Gateway Service](../system-services-gateway) configuration file like this:
```nginx title="dbrepo.conf"
location /api/upload {
allow 128.130.0.0/16;
deny all;
...
}
```
...@@ -2,22 +2,67 @@ ...@@ -2,22 +2,67 @@
author: Martin Weise author: Martin Weise
--- ---
## v1.7.0 (2025-03-03)
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.7.0)
!!! warning "Contains Breaking Changes"
This release updates the Metadata Database schema which is incompatible to v1.6.3! Follow the steps:
1. Make a backup of the database with `mariadb-dump`.
2. Apply the schema changes script: [`schema.sql`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.7/dbrepo-metadata-db/migration/16/schema.sql):
```shell
mariadb -h 127.0.0.1 -p3306 -u root --password=<password> -D dbrepo < schema.sql
```
3. Install the dependencies from the [`requirements.txt`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.7/dbrepo-metadata-db/migration/16/requirements.txt)
file or use your local environment:
```shell
pip install dbrepo==1.6.5rc15
```
4. Run the data migration script [`data.py`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.7/dbrepo-metadata-db/migration/16/data.py):
```shell
python data.py > data.sql
```
It generates the SQL statements used for migrating to the new schema.
5. Run the generated `data.sql` script:
```shell
mariadb -h 127.0.0.1 -p3306 -u root --password=<password> -D dbrepo < data.sql
```
#### Features
* Implemented a basic brute-force security defense strategy in the Auth Service that increments the wait time on wrong
logins in [#494](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/494).
* Implemented a password policy
in [#495](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/495).
#### Changes
* Replaced sequential numerical ids with non-guessable random ids in the Metadata Database
in [#491](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/491).
* Changed the interface for executing query in subsets/views
in [#493](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/493).
#### Removals
* Removed the Upload Service in favor of an internal stable upload endpoint in the Data Service
in [#492](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/492).
## v1.6.5 (2025-02-18) ## v1.6.5 (2025-02-18)
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.5) [:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.5)
### What's Changed
#### Fixes #### Fixes
* Fixed a bug where listing the views in the Python library did not work. * Fixed a bug where listing the views in the Python library did not work.
* Fixed a wrong MariaDB configuration where the `innodb_buffer_pool_size` variable was not configured to 70% of the
available memory in the Helm chart.
## v1.6.4 (2025-02-14) ## v1.6.4 (2025-02-14)
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.4) [:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.4)
### What's Changed
#### Fixes #### Fixes
* Fixed a bug where the users were not synced with the Metadata Database * Fixed a bug where the users were not synced with the Metadata Database
...@@ -27,8 +72,6 @@ author: Martin Weise ...@@ -27,8 +72,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.3) [:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.3)
### What's Changed
#### Changes #### Changes
* Refactored the UI to support OIDC and added an event listener to the Auth Service that syncs users on creation to the * Refactored the UI to support OIDC and added an event listener to the Auth Service that syncs users on creation to the
...@@ -38,8 +81,6 @@ author: Martin Weise ...@@ -38,8 +81,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.2) [:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.2)
### What's Changed
#### Changes #### Changes
* Added interface tests for the Python library in Gitlab CI/CD pipeline * Added interface tests for the Python library in Gitlab CI/CD pipeline
...@@ -54,8 +95,6 @@ author: Martin Weise ...@@ -54,8 +95,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.1) [:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.1)
### What's Changed
#### Changes #### Changes
* Added privacy feature for hidden databases (and optionally tables, views, subsets) that hides them completely from * Added privacy feature for hidden databases (and optionally tables, views, subsets) that hides them completely from
...@@ -70,8 +109,6 @@ author: Martin Weise ...@@ -70,8 +109,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.0) [:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.0)
### What's Changed
#### Features #### Features
* Added possibility to modify table description and privacy mode that hides metadata of databases, tables, subsets and * Added possibility to modify table description and privacy mode that hides metadata of databases, tables, subsets and
...@@ -109,8 +146,6 @@ author: Martin Weise ...@@ -109,8 +146,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.5.3) [:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.5.3)
### What's Changed
#### Fixes #### Fixes
* Fixed a bug where subsets containing sub-queries are not able to retrieve data * Fixed a bug where subsets containing sub-queries are not able to retrieve data
...@@ -120,8 +155,6 @@ author: Martin Weise ...@@ -120,8 +155,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.5.2) [:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.5.2)
### What's Changed
#### Changes #### Changes
* Adapt Helm chart to support `runAsNonRoot` throughout and specify `resource` presets for the highly-constrained * Adapt Helm chart to support `runAsNonRoot` throughout and specify `resource` presets for the highly-constrained
...@@ -141,8 +174,6 @@ author: Martin Weise ...@@ -141,8 +174,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.5.1) [:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.5.1)
### What's Changed
#### Fixes #### Fixes
* Bug where the data volume could not be calculated when the data length column in the Metadata Database is `null` * Bug where the data volume could not be calculated when the data length column in the Metadata Database is `null`
...@@ -160,8 +191,6 @@ author: Martin Weise ...@@ -160,8 +191,6 @@ author: Martin Weise
script [`schema_1.4.5-to-1.5.0.sql`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.5/dbrepo-metadata-db/migration/schema_1.4.5-to-1.5.0.sql) script [`schema_1.4.5-to-1.5.0.sql`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.5/dbrepo-metadata-db/migration/schema_1.4.5-to-1.5.0.sql)
to apply the changes manually. to apply the changes manually.
### What's Changed
#### Features #### Features
* Added `SERIAL` data type to create incrementing key * Added `SERIAL` data type to create incrementing key
...@@ -196,8 +225,6 @@ author: Martin Weise ...@@ -196,8 +225,6 @@ author: Martin Weise
This release updates the Metadata Database schema which is incompatible to v1.4.5! This release updates the Metadata Database schema which is incompatible to v1.4.5!
### What's Changed
#### Features #### Features
* Added [Dashboard Service](../api/dashboard-service/) and monitoring in default setup. * Added [Dashboard Service](../api/dashboard-service/) and monitoring in default setup.
......
.docs/images/screenshots/storage-service-filer.png

122 KiB

...@@ -14,7 +14,7 @@ author: Martin Weise ...@@ -14,7 +14,7 @@ author: Martin Weise
![Maintainability Rating](./images/maintainability.svg) ![Maintainability Rating](./images/maintainability.svg)
![Security Rating](./images/security.svg) ![Security Rating](./images/security.svg)
Documentation for version: [v1.6.5](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/releases). Documentation for version: [v1.7.0](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/releases).
DBRepo is a repository for data in databases that are used from the beginning until the end of a research DBRepo is a repository for data in databases that are used from the beginning until the end of a research
project supporting data evolution, -citation and -versioning. It implements the query store of the project supporting data evolution, -citation and -versioning. It implements the query store of the
......
...@@ -11,7 +11,7 @@ author: Martin Weise ...@@ -11,7 +11,7 @@ author: Martin Weise
If you have [Docker](https://docs.docker.com/engine/install/) already installed on your system, you can install DBRepo with: If you have [Docker](https://docs.docker.com/engine/install/) already installed on your system, you can install DBRepo with:
```shell ```shell
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.6/install.sh | bash curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.7/install.sh | bash
``` ```
!!! bug "Default installation security disclaimer" !!! bug "Default installation security disclaimer"
...@@ -38,7 +38,7 @@ SSL/TLS certificate is recommended. Follow the [secure installation](#secure-ins ...@@ -38,7 +38,7 @@ SSL/TLS certificate is recommended. Follow the [secure installation](#secure-ins
Execute the installation script to download only the environment and save it to `dist`. Execute the installation script to download only the environment and save it to `dist`.
```shell ```shell
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.6/install.sh | DOWNLOAD_ONLY=1 bash curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.7/install.sh | DOWNLOAD_ONLY=1 bash
``` ```
### Static Configuration ### Static Configuration
...@@ -82,7 +82,7 @@ the variable `IDENTITY_SERVICE_ADMIN_PASSWORD` in `.env`. ...@@ -82,7 +82,7 @@ the variable `IDENTITY_SERVICE_ADMIN_PASSWORD` in `.env`.
Update the client secret of the `dbrepo-client`: Update the client secret of the `dbrepo-client`:
```bash ```bash
curl -sSL "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.6/.scripts/reg-client-secret.sh" | bash curl -sSL "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.7/.scripts/reg-client-secret.sh" | bash
``` ```
Also, update the JWT key according to the Also, update the JWT key according to the
......
...@@ -6,7 +6,7 @@ author: Martin Weise ...@@ -6,7 +6,7 @@ author: Martin Weise
To install DBRepo in your existing cluster, download the To install DBRepo in your existing cluster, download the
sample [ sample [
`values.yaml`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.6/helm/dbrepo/values.yaml) `values.yaml`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.7/helm/dbrepo/values.yaml)
for your deployment and update the variables, especially `hostname`. for your deployment and update the variables, especially `hostname`.
```shell ```shell
...@@ -14,7 +14,7 @@ helm upgrade --install dbrepo \ ...@@ -14,7 +14,7 @@ helm upgrade --install dbrepo \
-n dbrepo \ -n dbrepo \
"oci://registry.datalab.tuwien.ac.at/dbrepo/helm/dbrepo" \ "oci://registry.datalab.tuwien.ac.at/dbrepo/helm/dbrepo" \
--values ./values.yaml \ --values ./values.yaml \
--version "1.6.5" \ --version "1.7.0" \
--create-namespace \ --create-namespace \
--cleanup-on-fail --cleanup-on-fail
``` ```
......
variables: variables:
BUILD_VERSION: ""
HOSTALIASES: "./hosts" HOSTALIASES: "./hosts"
DOCKER_HOST: "unix:///var/run/dind/docker.sock" DOCKER_HOST: "unix:///var/run/dind/docker.sock"
DOCKER_VERSION: "27" DOCKER_VERSION: "27"
...@@ -9,9 +10,9 @@ variables: ...@@ -9,9 +10,9 @@ variables:
NODE_VERSION: "18" NODE_VERSION: "18"
SONARQUBE_VERSION: "10.0" SONARQUBE_VERSION: "10.0"
BUN_VERSION: "1.1.40" BUN_VERSION: "1.1.40"
DOC_VERSION: "1.6" DOC_VERSION: "1.7"
APP_VERSION: "1.6.5" APP_VERSION: "1.7.0"
CHART_VERSION: "1.6.5" CHART_VERSION: "1.7.0"
CACHE_FALLBACK_KEY: "${CI_DEFAULT_BRANCH}" CACHE_FALLBACK_KEY: "${CI_DEFAULT_BRANCH}"
# This will supress any download for dependencies and plugins or upload messages which would clutter the console log. # This will supress any download for dependencies and plugins or upload messages which would clutter the console log.
# `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work. # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
...@@ -30,13 +31,20 @@ cache: ...@@ -30,13 +31,20 @@ cache:
paths: paths:
- .m2/repository - .m2/repository
workflow:
rules:
- if: $CI_COMMIT_REF_NAME == "dev"
variables:
BUILD_VERSION: "+git.${CI_COMMIT_SHORT_SHA}"
- when: always
stages: stages:
- lint - lint
- build - build
- deploy
- test - test
- docs - docs
- release - release
- deploy
- verify - verify
- scan - scan
...@@ -80,9 +88,9 @@ lint-helm-chart: ...@@ -80,9 +88,9 @@ lint-helm-chart:
- apk --no-cache add helm git bash - apk --no-cache add helm git bash
- cp ./helm/dbrepo/values.schema.json ./CI_values.schema.json - cp ./helm/dbrepo/values.schema.json ./CI_values.schema.json
- helm plugin install https://github.com/losisin/helm-values-schema-json.git - helm plugin install https://github.com/losisin/helm-values-schema-json.git
- helm dependency update ./helm/seaweedfs
- helm package ./helm/seaweedfs --destination ./build - helm package ./helm/seaweedfs --destination ./build
- helm dependency update ./helm/dbrepo - helm registry login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_URL}"
- helm registry login --username "${CI_REGISTRY2_USER}" --password "${CI_REGISTRY2_PASSWORD}" "${CI_REGISTRY2_URL}"
script: script:
- bash .scripts/check-helm.sh - bash .scripts/check-helm.sh
- helm lint ./helm/dbrepo - helm lint ./helm/dbrepo
...@@ -137,6 +145,9 @@ lint-open-api-version: ...@@ -137,6 +145,9 @@ lint-open-api-version:
build-metadata-service: build-metadata-service:
image: maven:3-openjdk-${JAVA_VERSION} image: maven:3-openjdk-${JAVA_VERSION}
stage: build stage: build
only:
- merge_requests
- master
script: script:
- "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests" - "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
# Compiled classes are needed for SonarQube in later stages # Compiled classes are needed for SonarQube in later stages
...@@ -155,6 +166,9 @@ build-metadata-service: ...@@ -155,6 +166,9 @@ build-metadata-service:
build-analyse-service: build-analyse-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: build stage: build
only:
- merge_requests
- master
variables: variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
script: script:
...@@ -164,6 +178,9 @@ build-analyse-service: ...@@ -164,6 +178,9 @@ build-analyse-service:
build-lib: build-lib:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: build stage: build
only:
- merge_requests
- master
variables: variables:
PIPENV_PIPFILE: "./lib/python/Pipfile" PIPENV_PIPFILE: "./lib/python/Pipfile"
script: script:
...@@ -173,12 +190,16 @@ build-lib: ...@@ -173,12 +190,16 @@ build-lib:
build-data-service: build-data-service:
image: maven:3-openjdk-${JAVA_VERSION} image: maven:3-openjdk-${JAVA_VERSION}
stage: build stage: build
only:
- merge_requests
- master
needs: needs:
- build-metadata-service - build-metadata-service
dependencies: dependencies:
- build-metadata-service - build-metadata-service
script: before_script:
- "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests" - "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
script:
- "mvn -f ./dbrepo-data-service/pom.xml clean package $MAVEN_OPTS -DskipTests" - "mvn -f ./dbrepo-data-service/pom.xml clean package $MAVEN_OPTS -DskipTests"
# Compiled classes are needed for SonarQube in later stages # Compiled classes are needed for SonarQube in later stages
artifacts: artifacts:
...@@ -192,19 +213,29 @@ build-data-service: ...@@ -192,19 +213,29 @@ build-data-service:
build-ui: build-ui:
image: oven/bun:${BUN_VERSION}-alpine image: oven/bun:${BUN_VERSION}-alpine
stage: build stage: build
only:
- merge_requests
- master
script: script:
- "cd ./dbrepo-ui && bun install && bun run build" - "cd ./dbrepo-ui && bun install && bun run build"
build-search-service: build-search-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: build stage: build
script: only:
- merge_requests
- master
before_script:
- "pip install pipenv" - "pip install pipenv"
script:
- "cd dbrepo-search-service && pipenv install --system --deploy" - "cd dbrepo-search-service && pipenv install --system --deploy"
build-images: build-images:
image: docker.io/docker:${DOCKER_VERSION}-dind image: docker.io/docker:${DOCKER_VERSION}-dind
stage: build stage: build
only:
- merge_requests
- master
before_script: before_script:
- "apk add --no-cache make" - "apk add --no-cache make"
- echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
...@@ -214,6 +245,10 @@ build-images: ...@@ -214,6 +245,10 @@ build-images:
build-helm: build-helm:
image: docker.io/docker:${DOCKER_VERSION}-dind image: docker.io/docker:${DOCKER_VERSION}-dind
stage: build stage: build
only:
- tags
- merge_requests
- master
before_script: before_script:
- apk add --no-cache helm make - apk add --no-cache helm make
- echo "$CI_GPG_KEYRING" | base64 -d > ./secring.gpg - echo "$CI_GPG_KEYRING" | base64 -d > ./secring.gpg
...@@ -221,30 +256,12 @@ build-helm: ...@@ -221,30 +256,12 @@ build-helm:
script: script:
- make build-helm - make build-helm
deploy-staging:
image: docker.io/alpine:${ALPINE_VERSION}
stage: deploy
variables:
KUBECONFIG: ./kubecfg
environment:
name: staging/datalab
url: ${CI_ENV_STAGING_URL}
before_script:
- apk add --no-cache helm kubectl make
- echo ${CI_K8S_CONFIG} | base64 -d > ./kubecfg
- make build-helm
- helm -n ${CI_ENV_STAGING_NAMESPACE} uninstall ${CI_ENV_STAGING_RELEASE_NAME} --ignore-not-found
- kubectl -n ${CI_ENV_STAGING_NAMESPACE} delete pvc --all
script:
- helm -n ${CI_ENV_STAGING_NAMESPACE} upgrade --install ${CI_ENV_STAGING_RELEASE_NAME} ./build/dbrepo-${CHART_VERSION}.tgz --create-namespace -f ./.gitlab/agents/dev/values.yaml
only:
refs:
- dev
- master
test-metadata-service: test-metadata-service:
image: maven:3-openjdk-${JAVA_VERSION} image: maven:3-openjdk-${JAVA_VERSION}
stage: test stage: test
only:
- merge_requests
- master
needs: needs:
- build-metadata-service - build-metadata-service
dependencies: dependencies:
...@@ -265,6 +282,9 @@ test-metadata-service: ...@@ -265,6 +282,9 @@ test-metadata-service:
test-data-service: test-data-service:
image: maven:3-openjdk-${JAVA_VERSION} image: maven:3-openjdk-${JAVA_VERSION}
stage: test stage: test
only:
- merge_requests
- master
needs: needs:
- build-data-service - build-data-service
dependencies: dependencies:
...@@ -283,16 +303,12 @@ test-data-service: ...@@ -283,16 +303,12 @@ test-data-service:
junit: ./dbrepo-data-service/rest-service/target/surefire-reports/TEST-*.xml junit: ./dbrepo-data-service/rest-service/target/surefire-reports/TEST-*.xml
coverage: '/Total.*?([0-9]{1,3})%/' coverage: '/Total.*?([0-9]{1,3})%/'
test-upload-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: test
script:
- "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
- "mvn -f ./dbrepo-upload-service/pom.xml clean test $MAVEN_OPTS"
test-analyse-service: test-analyse-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test stage: test
only:
- merge_requests
- master
variables: variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
needs: needs:
...@@ -302,21 +318,24 @@ test-analyse-service: ...@@ -302,21 +318,24 @@ test-analyse-service:
script: script:
- "pip install pipenv" - "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy" - "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./dbrepo-analyse-service/ && coverage run -m pytest tests/test_determine_dt.py tests/test_determine_pk.py tests/test_s3_client.py --junitxml=report.xml && coverage html --omit="tests/*" && coverage report --omit="tests/*" > ./coverage.txt - cd ./dbrepo-analyse-service/ && coverage run --rcfile=.coveragerc -m pytest tests/test_determine_dt.py tests/test_determine_pk.py tests/test_s3_client.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts: artifacts:
when: always when: always
paths: paths:
- ./dbrepo-analyse-service/report.xml - ./dbrepo-analyse-service/coverage.xml
- ./dbrepo-analyse-service/coverage.txt - ./dbrepo-analyse-service/coverage.txt
expire_in: 1 days expire_in: 1 days
reports: reports:
junit: ./dbrepo-analyse-service/report.xml junit: ./dbrepo-analyse-service/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/' coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-auth-service-init: test-auth-service-init:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test stage: test
only:
- merge_requests
- master
variables: variables:
PIPENV_PIPFILE: "./dbrepo-auth-service/init/Pipfile" PIPENV_PIPFILE: "./dbrepo-auth-service/init/Pipfile"
before_script: before_script:
...@@ -324,21 +343,24 @@ test-auth-service-init: ...@@ -324,21 +343,24 @@ test-auth-service-init:
- "pip install pipenv" - "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy" - "pipenv install gunicorn && pipenv install --dev --system --deploy"
script: script:
- cd ./dbrepo-auth-service/init/ && coverage run -m pytest tests/test_unit_app.py --junitxml=report.xml && coverage html --omit="tests/*" && coverage report --omit="tests/*" > ./coverage.txt - cd ./dbrepo-auth-service/init/ && coverage run --rcfile=.coveragerc -m pytest tests/test_unit_app.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts: artifacts:
when: always when: always
paths: paths:
- ./dbrepo-auth-service/init/report.xml - ./dbrepo-auth-service/init/coverage.xml
- ./dbrepo-auth-service/init/coverage.txt - ./dbrepo-auth-service/init/coverage.txt
expire_in: 1 days expire_in: 1 days
reports: reports:
junit: ./dbrepo-auth-service/init/report.xml junit: ./dbrepo-auth-service/init/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/' coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-search-service: test-search-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test stage: test
only:
- merge_requests
- master
variables: variables:
PIPENV_PIPFILE: "./dbrepo-search-service/Pipfile" PIPENV_PIPFILE: "./dbrepo-search-service/Pipfile"
needs: needs:
...@@ -355,21 +377,24 @@ test-search-service: ...@@ -355,21 +377,24 @@ test-search-service:
script: script:
- "pip install pipenv" - "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy" - "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./dbrepo-search-service/ && coverage run -m pytest tests/test_app.py tests/test_jwt.py tests/test_opensearch_client.py tests/test_keycloak_client.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt - cd ./dbrepo-search-service/ && coverage run --rcfile=.coveragerc -m pytest tests/test_app.py tests/test_jwt.py tests/test_opensearch_client.py tests/test_keycloak_client.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts: artifacts:
when: always when: always
paths: paths:
- ./dbrepo-search-service/report.xml - ./dbrepo-search-service/coverage.xml
- ./dbrepo-search-service/coverage.txt - ./dbrepo-search-service/coverage.txt
expire_in: 1 days expire_in: 1 days
reports: reports:
junit: ./dbrepo-search-service/report.xml junit: ./dbrepo-search-service/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/' coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-search-service-init: test-search-service-init:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test stage: test
only:
- merge_requests
- master
variables: variables:
PIPENV_PIPFILE: "./dbrepo-search-service/init/Pipfile" PIPENV_PIPFILE: "./dbrepo-search-service/init/Pipfile"
needs: needs:
...@@ -379,21 +404,24 @@ test-search-service-init: ...@@ -379,21 +404,24 @@ test-search-service-init:
script: script:
- "pip install pipenv" - "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy" - "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./dbrepo-search-service/init/ && coverage run -m pytest tests/test_app.py tests/test_keycloak_client.py tests/test_opensearch_client.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt - cd ./dbrepo-search-service/init/ && coverage run --rcfile=.coveragerc -m pytest tests/test_app.py tests/test_keycloak_client.py tests/test_opensearch_client.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts: artifacts:
when: always when: always
paths: paths:
- ./dbrepo-search-service/report.xml - ./dbrepo-search-service/coverage.xml
- ./dbrepo-search-service/coverage.txt - ./dbrepo-search-service/coverage.txt
expire_in: 1 days expire_in: 1 days
reports: reports:
junit: ./dbrepo-search-service/report.xml junit: ./dbrepo-search-service/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/' coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-lib: test-lib:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test stage: test
only:
- merge_requests
- master
variables: variables:
PIPENV_PIPFILE: "./lib/python/Pipfile" PIPENV_PIPFILE: "./lib/python/Pipfile"
needs: needs:
...@@ -403,21 +431,24 @@ test-lib: ...@@ -403,21 +431,24 @@ test-lib:
script: script:
- "pip install pipenv" - "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy" - "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./lib/python/ && coverage run -m pytest tests/test_unit_analyse.py tests/test_unit_container.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py tests/test_unit_rest_client.py --junitxml=report.xml && coverage html --omit="tests/*" && coverage report --omit="tests/*" > ./coverage.txt - cd ./lib/python/ && coverage run --rcfile=.coveragerc -m pytest tests/test_unit_container.py tests/test_unit_messages.py tests/test_unit_image.py tests/test_unit_concept.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py tests/test_unit_unit.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts: artifacts:
when: always when: always
paths: paths:
- ./lib/python/report.xml - ./lib/python/coverage.xml
- ./lib/python/coverage.txt - ./lib/python/coverage.txt
expire_in: 1 days expire_in: 1 days
reports: reports:
junit: ./lib/python/report.xml junit: ./lib/python/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/' coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-ui: test-ui:
stage: test
image: docker.io/docker:${DOCKER_VERSION}-dind image: docker.io/docker:${DOCKER_VERSION}-dind
stage: test
only:
- merge_requests
- master
needs: needs:
- build-ui - build-ui
dependencies: dependencies:
...@@ -433,13 +464,18 @@ test-ui: ...@@ -433,13 +464,18 @@ test-ui:
release-images: release-images:
stage: release stage: release
image: docker:${DOCKER_VERSION}-dind image: docker:${DOCKER_VERSION}-dind
only:
- dev
- tags
dependencies: dependencies:
- test-metadata-service
- test-data-service
- test-analyse-service - test-analyse-service
only: - test-auth-service-init
refs: - test-data-service
- /^release-.*/ - test-lib
- test-metadata-service
- test-search-service
- test-search-service-init
- test-ui
before_script: before_script:
- "docker logout ${CI_REGISTRY_URL}" - "docker logout ${CI_REGISTRY_URL}"
- "echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY_URL}" - "echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY_URL}"
...@@ -454,9 +490,7 @@ release-helm: ...@@ -454,9 +490,7 @@ release-helm:
stage: release stage: release
image: docker:${DOCKER_VERSION}-dind image: docker:${DOCKER_VERSION}-dind
only: only:
refs: - tags
- /^release-.*/
when: manual
needs: needs:
- build-helm - build-helm
dependencies: dependencies:
...@@ -470,6 +504,8 @@ release-helm: ...@@ -470,6 +504,8 @@ release-helm:
- "mkdir -p ~/.gnupg" - "mkdir -p ~/.gnupg"
- echo "$CI_GPG_KEYRING" | base64 -d > ~/.gnupg/secring.gpg - echo "$CI_GPG_KEYRING" | base64 -d > ~/.gnupg/secring.gpg
- echo "$CI_GPG_KEYRING2" | base64 -d > ~/.gnupg/pubring.gpg - echo "$CI_GPG_KEYRING2" | base64 -d > ~/.gnupg/pubring.gpg
- helm registry login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_URL}"
- helm registry login --username "${CI_REGISTRY2_USER}" --password "${CI_REGISTRY2_PASSWORD}" "${CI_REGISTRY2_URL}"
- make build-helm - make build-helm
- "helm package ./helm/dbrepo --sign --key 'Martin Weise' --keyring ~/.gnupg/secring.gpg --destination ./build" - "helm package ./helm/dbrepo --sign --key 'Martin Weise' --keyring ~/.gnupg/secring.gpg --destination ./build"
- "helm plugin install https://github.com/sigstore/helm-sigstore" - "helm plugin install https://github.com/sigstore/helm-sigstore"
...@@ -481,8 +517,7 @@ release-docs: ...@@ -481,8 +517,7 @@ release-docs:
stage: release stage: release
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
only: only:
refs: - tags
- /^release-.*/
before_script: before_script:
- "apk add --no-cache alpine-sdk bash sed wget openssh jq curl" - "apk add --no-cache alpine-sdk bash sed wget openssh jq curl"
- "pip install pipenv" - "pip install pipenv"
...@@ -514,10 +549,8 @@ release-docs: ...@@ -514,10 +549,8 @@ release-docs:
release-libs: release-libs:
stage: release stage: release
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
when: manual
only: only:
refs: - tags
- /^release-.*/
variables: variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile" PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
before_script: before_script:
...@@ -532,8 +565,7 @@ verify-install-script: ...@@ -532,8 +565,7 @@ verify-install-script:
image: docker.io/docker:${DOCKER_VERSION}-dind image: docker.io/docker:${DOCKER_VERSION}-dind
stage: verify stage: verify
only: only:
refs: - tags
- /^release-.*/
variables: variables:
SKIP_CHECKS: 1 SKIP_CHECKS: 1
before_script: before_script:
...@@ -545,25 +577,37 @@ verify-dist: ...@@ -545,25 +577,37 @@ verify-dist:
image: docker.io/alpine:${ALPINE_VERSION} image: docker.io/alpine:${ALPINE_VERSION}
stage: verify stage: verify
only: only:
refs: - tags
- /^release-.*/
before_script: before_script:
- "apk add curl" - "apk add curl"
script: script:
- "curl -v --output /dev/null --fail https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${APP_VERSION}/dist.tar.gz" - "curl -v --output /dev/null --fail https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${APP_VERSION}/dist.tar.gz"
deploy-staging:
image: docker.io/alpine:${ALPINE_VERSION}
stage: deploy
only:
- dev
variables:
KUBECONFIG: ./kubecfg
environment:
name: staging/datalab
url: ${CI_ENV_STAGING_URL}
before_script:
- apk add --no-cache helm kubectl make
- echo ${CI_K8S_CONFIG} | base64 -d > ./kubecfg
- make build-helm
- helm -n ${CI_ENV_STAGING_NAMESPACE} uninstall ${CI_ENV_STAGING_RELEASE_NAME} --ignore-not-found
- kubectl -n ${CI_ENV_STAGING_NAMESPACE} delete pvc --all
- echo "${CI_K8S_VALUES}" | base64 -d > ./values.yaml
script:
- helm -n ${CI_ENV_STAGING_NAMESPACE} upgrade --install ${CI_ENV_STAGING_RELEASE_NAME} ./build/dbrepo-${CHART_VERSION}.tgz --create-namespace -f ./values.yaml
scan-sonarqube: scan-sonarqube:
image: sonarsource/sonar-scanner-cli:${SONARQUBE_VERSION} image: sonarsource/sonar-scanner-cli:${SONARQUBE_VERSION}
stage: scan stage: scan
only: only:
refs: - master
- master
needs:
- build-data-service
- build-metadata-service
dependencies:
- build-data-service
- build-metadata-service
script: script:
- 'sonar-scanner -Dsonar.token="${CI_SONAR_TOKEN}"' - 'sonar-scanner -Dsonar.token="${CI_SONAR_TOKEN}"'
allow_failure: true allow_failure: true
......
hostname: s155.datalab.tuwien.ac.at
gateway: https://s155.datalab.tuwien.ac.at
metadatadb:
enabled: true
rootUser:
user: root
password: da19c7cf5c0deba7bd47c174a0eb273b
galera:
mariabackup:
user: mariabackup
password: 9e447eeaf3e4b6aa26ea01582f0e8a54
persistence:
enabled: true
authservice:
enabled: true
auth:
adminUser: admin
adminPassword: ea72038fa14b968fc0ed09e182ecf624
postgresql:
auth:
postgresPassword: 129d5b888b8df271fa482da39f15c513
jwt:
pubkey: "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB"
client:
id: dbrepo-client
secret: MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG
setupJob:
image:
name: registry.datalab.tuwien.ac.at/dbrepo/auth-service-init:1.6.3
persistence:
enabled: true
brokerservice:
enabled: true
ldap:
bindpw: b8534187c9adf9618e7bd1c79c7f4639
identityservice:
enabled: true
global:
adminUser: admin
adminPassword: b8534187c9adf9618e7bd1c79c7f4639
users: admin
userPasswords: ea72038fa14b968fc0ed09e182ecf624
datadb:
enabled: true
rootUser:
user: root
password: fdf8578499b2083eb3aa03a861ac7912
galera:
mariabackup:
user: mariabackup
password: ef60e32e3217525474635cd28422c829
replicaCount: 3
persistence:
enabled: true
searchdb:
enabled: true
security:
enabled: false
extraEnvs:
- name: DISABLE_INSTALL_DEMO_CONFIG
value: "true"
persistence:
enabled: true
analyseservice:
enabled: true
image:
name: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.6.3
metadataservice:
enabled: true
image:
name: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.6.3
admin:
email: noreply@example.com
deletedRecord: permanent
repositoryName: Database Repository
granularity: YYYY-MM-DDThh:mm:ssZ
datacite:
enabled: false
url: https://api.datacite.org
prefix: ""
username: ""
password: ""
dataservice:
enabled: true
image:
name: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.6.3
rabbitmq:
consumer:
username: admin
password: ea72038fa14b968fc0ed09e182ecf624
s3:
auth:
username: a45e7a77607a8906e92237f00ea72f58
password: e2c4303dcbfd3a2c606fe30d19fcb82b
filePath: /s3
searchservice:
enabled: true
image:
name: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.6.3
init:
image:
name: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.6.3
storageservice:
enabled: true
init:
image:
name: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.6.3
uploadservice:
enabled: true
dashboardservice:
enabled: true
metricdb:
enabled: true
server:
rbac:
create: false
ui:
enabled: true
image:
name: registry.datalab.tuwien.ac.at/dbrepo/ui:1.6.3
public:
api:
client: https://s155.datalab.tuwien.ac.at
server: https://s155.datalab.tuwien.ac.at
title: "Database Repository"
logo: "https://ec.tuwien.ac.at/~weise/images/DS_white_hiRes.png"
icon: "https://ec.tuwien.ac.at/~weise/images/favicon.ico"
touch: "https://ec.tuwien.ac.at/~weise/images/DS-icon_white_hiRes.png"
broker:
host: s155.datalab.tuwien.ac.at
extra: "128.130.0.0/15"
database:
extra: "128.130.0.0/15"
pid:
default:
publisher: "TU Wien"
doi:
enabled: false
endpoint: https://doi.org
extraVolumes: [ ]
# - name: images-map
# configMap:
# name: ui-config
extraVolumeMounts: [ ]
# - name: images-map
# mountPath: /static/logo.svg
# subPath: logo.svg
gatewayservice:
enabled: true
ingress:
enabled: true
className: nginx
tls:
enabled: true
secretName: ingress-cert
annotations:
cert-manager.io/cluster-issuer: letsencrypt-cluster-issuer
# nginx.ingress.kubernetes.io/whitelist-source-range: 128.130.0.0/15
.PHONY: all .PHONY: all
APP_VERSION ?= 1.6.5 APP_VERSION ?= 1.7.0
CHART_VERSION ?= 1.6.5 CHART_VERSION ?= 1.7.0
REPOSITORY_URL ?= registry.datalab.tuwien.ac.at/dbrepo REPOSITORY_URL ?= registry.datalab.tuwien.ac.at/dbrepo
.PHONY: all .PHONY: all
......
...@@ -14,7 +14,7 @@ If you have [Docker](https://docs.docker.com/engine/install/) already installed ...@@ -14,7 +14,7 @@ If you have [Docker](https://docs.docker.com/engine/install/) already installed
with: with:
```bash ```bash
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.6/install.sh | bash curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.7/install.sh | bash
``` ```
## Documentation ## Documentation
......
[report]
format = text
omit =
# omit tests
./tests/*
# omit ext lib
./omlib/*
[html]
directory = htmlcov
\ No newline at end of file
...@@ -21,7 +21,7 @@ numpy = "*" ...@@ -21,7 +21,7 @@ numpy = "*"
pandas = "*" pandas = "*"
minio = "*" minio = "*"
pydantic = "*" pydantic = "*"
dbrepo = {path = "./lib/dbrepo-1.6.5.tar.gz"} dbrepo = {path = "./lib/dbrepo-1.7.0.tar.gz"}
opensearch-py = "*" opensearch-py = "*"
[dev-packages] [dev-packages]
......
This diff is collapsed.