Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • fair-data-austria-db-repository/fda-services
1 result
Select Git revision
Show changes
Commits on Source (12)
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 6430 additions and 6205 deletions
.docker/docker-compose.yml
View file @ 841d4894
......@@ -3,7 +3,6 @@ volumes:
data-db-data:
auth-db-data:
broker-service-data:
upload-service-data:
search-db-data:
identity-service-data:
metric-db-data:
......@@ -114,7 +113,7 @@ services:
init: true
restart: "no"
container_name: dbrepo-auth-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.6.5
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.7.0
environment:
AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-admin}
AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-admin}
......@@ -135,7 +134,7 @@ services:
restart: "no"
container_name: dbrepo-metadata-service
hostname: metadata-service
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.6.5
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.7.0
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
......@@ -198,7 +197,7 @@ services:
restart: "no"
container_name: dbrepo-analyse-service
hostname: analyse-service
image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.6.5
image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.7.0
environment:
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
......@@ -253,7 +252,7 @@ services:
restart: "no"
container_name: dbrepo-search-db
hostname: search-db
image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.6.5
image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.7.0
healthcheck:
test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP
interval: 10s
......@@ -277,7 +276,7 @@ services:
restart: "no"
container_name: dbrepo-search-service
hostname: search-service
image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.6.5
image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.7.0
environment:
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT_SECRET:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
......@@ -301,11 +300,10 @@ services:
restart: "no"
container_name: dbrepo-ui
hostname: ui
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.6.5
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.7.0
environment:
NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}"
NUXT_PUBLIC_API_SERVER: "${BASE_URL:-http://gateway-service}"
NUXT_PUBLIC_UPLOAD_CLIENT: "${BASE_URL:-http://localhost}/api/upload/files"
NUXT_OIDC_PROVIDERS_KEYCLOAK_AUTHORIZATION_URL: "${BASE_URL:-http://localhost}/realms/dbrepo/protocol/openid-connect/auth"
NUXT_OIDC_PROVIDERS_KEYCLOAK_BASE_URL: "${BASE_URL:-http://localhost}/realms/dbrepo"
NUXT_OIDC_PROVIDERS_KEYCLOAK_CLIENT_ID: "${AUTH_SERVICE_CLIENT:-dbrepo-client}"
......@@ -318,8 +316,6 @@ services:
depends_on:
dbrepo-search-service:
condition: service_healthy
dbrepo-upload-service:
condition: service_healthy
healthcheck:
test: curl -fsSL http://127.0.0.1:3000 && curl -fsSL http://127.0.0.1:3000/health
interval: 10s
......@@ -381,7 +377,7 @@ services:
init: true
container_name: dbrepo-search-service-init
hostname: search-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.6.5
image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.7.0
environment:
LOG_LEVEL: ${LOG_LEVEL:-info}
METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
......@@ -438,7 +434,7 @@ services:
restart: "no"
container_name: dbrepo-dashboard-service
hostname: dashboard-service
image: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.6.5
image: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.7.0
ports:
- "3000:3000"
volumes:
......@@ -465,7 +461,7 @@ services:
init: true
container_name: dbrepo-storage-service-init
hostname: storage-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.6.5
image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.7.0
environment:
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-seaweedfsadmin}
S3_BUCKET: "${S3_BUCKET:-dbrepo}"
......@@ -477,41 +473,11 @@ services:
logging:
driver: json-file
dbrepo-upload-service:
restart: "no"
container_name: dbrepo-upload-service
hostname: upload-service
image: docker.io/tusproject/tusd:v2.4.0
volumes:
- "./config/pre-create.sh:/srv/tusd-hooks/pre-create:ro"
command:
- "-behind-proxy"
- "-max-size=2000000000"
- "-base-path=/api/upload/files/"
- "-hooks-dir=/srv/tusd-hooks/"
- "-s3-endpoint=${STORAGE_ENDPOINT:-http://storage-service:9000}"
- "-s3-bucket=dbrepo"
environment:
AWS_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
AWS_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
AWS_REGION: "${STORAGE_REGION_NAME:-default}"
METADATA_SERVICE_ENDPOINT: "${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}"
depends_on:
dbrepo-storage-service:
condition: service_healthy
healthcheck:
test: wget -qO- localhost:8080/metrics | grep "tusd" || exit 1
interval: 10s
timeout: 5s
retries: 12
logging:
driver: json-file
dbrepo-data-service:
restart: "no"
container_name: dbrepo-data-service
hostname: data-service
image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.6.5
image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.7.0
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
......
.docs/.openapi/api-data.yaml
View file @ 841d4894
This diff is collapsed.
.docs/.openapi/api-metadata.yaml
View file @ 841d4894
This diff is collapsed.
.docs/.openapi/api.base.yaml
View file @ 841d4894
......@@ -11,7 +11,7 @@ components:
type: http
externalDocs:
description: Project Website
url: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.6/
url: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.7/
info:
contact:
email: andreas.rauber@tuwien.ac.at
......@@ -24,7 +24,7 @@ info:
name: Apache 2.0
url: https://www.apache.org/licenses/LICENSE-2.0
title: DBRepo REST API
version: 1.6.5
version: 1.7.0
openapi: 3.1.0
servers:
- description: Test Instance
......
.docs/.openapi/api.yaml
View file @ 841d4894
This diff is collapsed.
.docs/api/storage-service.md
View file @ 841d4894
......@@ -22,19 +22,23 @@ author: Martin Weise
We use [SeaweedFS](https://seaweedfs.github.io/) as a high-performance, S3 compatible object store for easy, cloud-ready
deployments that by default support replication and monitoring. No graphical user interface is provided out-of-the-box,
administrators can access the S3 storage via S3-compatible clients
administrators can access the S3 storage via S3-compatible clients
e.g. [AWS CLI](https://docs.aws.amazon.com/cli/latest/reference/s3/) (see below).
### Users
The default configuration creates admin credentials `seaweedfsadmin:seaweedfsadmin`. By default, one bucket `dbrepo` is
created that holds uploads temporarily. It is recommended to delete the contents regularly.
The default configuration creates one user `seaweedfsadmin` with password `seaweedfsadmin`.
The S3 endpoint of the Storage Service is available on port `9000`.
### Buckets
### Filer UI
The default configuration creates two buckets `dbrepo-upload`, `dbrepo-download`:
The storage service comes with a simple UI that can be used to explore the uploaded files, rename them and delete them.
Please note that the Filer UI is not intended for production and should be turned off for security purposes.
* `dbrepo-upload` for CSV-file upload (for import of data, analysis, etc.) from the User Interface
* `dbrepo-download` for CSV-file download (exporting data, metadata, etc.)
<figure markdown>
![Filer UI with a list of uploaded files in the bucket dbrepo](../images/screenshots/storage-service-filer.png)
<figcaption>Figure 1: Filer UI</figcaption>
</figure>
## Limitations
......
.docs/api/upload-service.md deleted 100644 → 0
View file @ 79a3df8a
---
author: Martin Weise
---
## tl;dr
!!! debug "Debug Information"
Image: [`docker.io/tusproject/tusd:v1.12`](https://hub.docker.com/r/tusproject/tusd)
* Ports: 1080/tcp
* Prometheus: `http://<hostname>:1080/api/upload/metrics`
* API: `http://<hostname>:1080/api/upload`
To directly access in Kubernetes (for e.g. debugging), forward the svc port to your local machine:
```shell
kubectl [-n namespace] port-forward svc/upload-service 1080:80
```
## Overview
We use the [TUS](https://tus.io/) open protocol for resume-able file uploads which based entirely on HTTP. Even though
the Upload Service is part of the standard installation, it is an entirely optional component and can be replaced with
any S3-compatible Blob Storage.
### Architecture
The Upload Service communicates internally with the [Storage Service](../storage-service) (c.f. [Figure 1](#fig1)).
<figure id="fig1" markdown>
![Architecture of the Upload Service](../images/architecture-upload-service.svg)
<figcaption>Figure 1: Architecture of the Upload Service</figcaption>
</figure>
The Upload Service is responsible for uploading files (mainly CSV-files) into a Blob Storage that can be accesses trough
the S3 protocol (e.g. our [Storage Service](../storage-service)). Make sure that the Upload Service can be
accessed from the Gateway Service.
## Limitations
* No support for authentication.
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
## Security
1. We strongly encourage to limit the clients allowed to upload by adding your subnet, e.g. `128.130.0.0/16`
(=TU Wien subnet) to the [Gateway Service](../system-services-gateway) configuration file like this:
```nginx title="dbrepo.conf"
location /api/upload {
allow 128.130.0.0/16;
deny all;
...
}
```
.docs/changelog.md
View file @ 841d4894
......@@ -2,22 +2,67 @@
author: Martin Weise
---
## v1.7.0 (2025-03-03)
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.7.0)
!!! warning "Contains Breaking Changes"
This release updates the Metadata Database schema which is incompatible to v1.6.3! Follow the steps:
1. Make a backup of the database with `mariadb-dump`.
2. Apply the schema changes script: [`schema.sql`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.7/dbrepo-metadata-db/migration/16/schema.sql):
```shell
mariadb -h 127.0.0.1 -p3306 -u root --password=<password> -D dbrepo < schema.sql
```
3. Install the dependencies from the [`requirements.txt`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.7/dbrepo-metadata-db/migration/16/requirements.txt)
file or use your local environment:
```shell
pip install dbrepo==1.6.5rc15
```
4. Run the data migration script [`data.py`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.7/dbrepo-metadata-db/migration/16/data.py):
```shell
python data.py > data.sql
```
It generates the SQL statements used for migrating to the new schema.
5. Run the generated `data.sql` script:
```shell
mariadb -h 127.0.0.1 -p3306 -u root --password=<password> -D dbrepo < data.sql
```
#### Features
* Implemented a basic brute-force security defense strategy in the Auth Service that increments the wait time on wrong
logins in [#494](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/494).
* Implemented a password policy
in [#495](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/495).
#### Changes
* Replaced sequential numerical ids with non-guessable random ids in the Metadata Database
in [#491](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/491).
* Changed the interface for executing query in subsets/views
in [#493](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/493).
#### Removals
* Removed the Upload Service in favor of an internal stable upload endpoint in the Data Service
in [#492](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/issues/492).
## v1.6.5 (2025-02-18)
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.5)
### What's Changed
#### Fixes
* Fixed a bug where listing the views in the Python library did not work.
* Fixed a wrong MariaDB configuration where the `innodb_buffer_pool_size` variable was not configured to 70% of the
available memory in the Helm chart.
## v1.6.4 (2025-02-14)
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.4)
### What's Changed
#### Fixes
* Fixed a bug where the users were not synced with the Metadata Database
......@@ -27,8 +72,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.3)
### What's Changed
#### Changes
* Refactored the UI to support OIDC and added an event listener to the Auth Service that syncs users on creation to the
......@@ -38,8 +81,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.2)
### What's Changed
#### Changes
* Added interface tests for the Python library in Gitlab CI/CD pipeline
......@@ -54,8 +95,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.1)
### What's Changed
#### Changes
* Added privacy feature for hidden databases (and optionally tables, views, subsets) that hides them completely from
......@@ -70,8 +109,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.6.0)
### What's Changed
#### Features
* Added possibility to modify table description and privacy mode that hides metadata of databases, tables, subsets and
......@@ -109,8 +146,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.5.3)
### What's Changed
#### Fixes
* Fixed a bug where subsets containing sub-queries are not able to retrieve data
......@@ -120,8 +155,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.5.2)
### What's Changed
#### Changes
* Adapt Helm chart to support `runAsNonRoot` throughout and specify `resource` presets for the highly-constrained
......@@ -141,8 +174,6 @@ author: Martin Weise
[:simple-gitlab: GitLab Release](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tags/v1.5.1)
### What's Changed
#### Fixes
* Bug where the data volume could not be calculated when the data length column in the Metadata Database is `null`
......@@ -160,8 +191,6 @@ author: Martin Weise
script [`schema_1.4.5-to-1.5.0.sql`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.5/dbrepo-metadata-db/migration/schema_1.4.5-to-1.5.0.sql)
to apply the changes manually.
### What's Changed
#### Features
* Added `SERIAL` data type to create incrementing key
......@@ -196,8 +225,6 @@ author: Martin Weise
This release updates the Metadata Database schema which is incompatible to v1.4.5!
### What's Changed
#### Features
* Added [Dashboard Service](../api/dashboard-service/) and monitoring in default setup.
......
.docs/images/screenshots/storage-service-filer.png 0 → 100644
View file @ 841d4894
.docs/images/screenshots/storage-service-filer.png

122 KiB

.docs/index.md
View file @ 841d4894
......@@ -14,7 +14,7 @@ author: Martin Weise
![Maintainability Rating](./images/maintainability.svg)
![Security Rating](./images/security.svg)
Documentation for version: [v1.6.5](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/releases).
Documentation for version: [v1.7.0](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/releases).
DBRepo is a repository for data in databases that are used from the beginning until the end of a research
project supporting data evolution, -citation and -versioning. It implements the query store of the
......
.docs/installation.md
View file @ 841d4894
......@@ -11,7 +11,7 @@ author: Martin Weise
If you have [Docker](https://docs.docker.com/engine/install/) already installed on your system, you can install DBRepo with:
```shell
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.6/install.sh | bash
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.7/install.sh | bash
```
!!! bug "Default installation security disclaimer"
......@@ -38,7 +38,7 @@ SSL/TLS certificate is recommended. Follow the [secure installation](#secure-ins
Execute the installation script to download only the environment and save it to `dist`.
```shell
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.6/install.sh | DOWNLOAD_ONLY=1 bash
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.7/install.sh | DOWNLOAD_ONLY=1 bash
```
### Static Configuration
......@@ -82,7 +82,7 @@ the variable `IDENTITY_SERVICE_ADMIN_PASSWORD` in `.env`.
Update the client secret of the `dbrepo-client`:
```bash
curl -sSL "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.6/.scripts/reg-client-secret.sh" | bash
curl -sSL "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.7/.scripts/reg-client-secret.sh" | bash
```
Also, update the JWT key according to the
......
.docs/kubernetes.md
View file @ 841d4894
......@@ -6,7 +6,7 @@ author: Martin Weise
To install DBRepo in your existing cluster, download the
sample [
`values.yaml`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.6/helm/dbrepo/values.yaml)
`values.yaml`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.7/helm/dbrepo/values.yaml)
for your deployment and update the variables, especially `hostname`.
```shell
......@@ -14,7 +14,7 @@ helm upgrade --install dbrepo \
-n dbrepo \
"oci://registry.datalab.tuwien.ac.at/dbrepo/helm/dbrepo" \
--values ./values.yaml \
--version "1.6.5" \
--version "1.7.0" \
--create-namespace \
--cleanup-on-fail
```
......
.gitlab-ci.yml
View file @ 841d4894
variables:
BUILD_VERSION: ""
HOSTALIASES: "./hosts"
DOCKER_HOST: "unix:///var/run/dind/docker.sock"
DOCKER_VERSION: "27"
......@@ -9,9 +10,9 @@ variables:
NODE_VERSION: "18"
SONARQUBE_VERSION: "10.0"
BUN_VERSION: "1.1.40"
DOC_VERSION: "1.6"
APP_VERSION: "1.6.5"
CHART_VERSION: "1.6.5"
DOC_VERSION: "1.7"
APP_VERSION: "1.7.0"
CHART_VERSION: "1.7.0"
CACHE_FALLBACK_KEY: "${CI_DEFAULT_BRANCH}"
# This will supress any download for dependencies and plugins or upload messages which would clutter the console log.
# `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work.
......@@ -30,13 +31,20 @@ cache:
paths:
- .m2/repository
workflow:
rules:
- if: $CI_COMMIT_REF_NAME == "dev"
variables:
BUILD_VERSION: "+git.${CI_COMMIT_SHORT_SHA}"
- when: always
stages:
- lint
- build
- deploy
- test
- docs
- release
- deploy
- verify
- scan
......@@ -80,9 +88,9 @@ lint-helm-chart:
- apk --no-cache add helm git bash
- cp ./helm/dbrepo/values.schema.json ./CI_values.schema.json
- helm plugin install https://github.com/losisin/helm-values-schema-json.git
- helm dependency update ./helm/seaweedfs
- helm package ./helm/seaweedfs --destination ./build
- helm dependency update ./helm/dbrepo
- helm registry login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_URL}"
- helm registry login --username "${CI_REGISTRY2_USER}" --password "${CI_REGISTRY2_PASSWORD}" "${CI_REGISTRY2_URL}"
script:
- bash .scripts/check-helm.sh
- helm lint ./helm/dbrepo
......@@ -137,6 +145,9 @@ lint-open-api-version:
build-metadata-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: build
only:
- merge_requests
- master
script:
- "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
# Compiled classes are needed for SonarQube in later stages
......@@ -155,6 +166,9 @@ build-metadata-service:
build-analyse-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: build
only:
- merge_requests
- master
variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
script:
......@@ -164,6 +178,9 @@ build-analyse-service:
build-lib:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: build
only:
- merge_requests
- master
variables:
PIPENV_PIPFILE: "./lib/python/Pipfile"
script:
......@@ -173,12 +190,16 @@ build-lib:
build-data-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: build
only:
- merge_requests
- master
needs:
- build-metadata-service
dependencies:
- build-metadata-service
script:
before_script:
- "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
script:
- "mvn -f ./dbrepo-data-service/pom.xml clean package $MAVEN_OPTS -DskipTests"
# Compiled classes are needed for SonarQube in later stages
artifacts:
......@@ -192,19 +213,29 @@ build-data-service:
build-ui:
image: oven/bun:${BUN_VERSION}-alpine
stage: build
only:
- merge_requests
- master
script:
- "cd ./dbrepo-ui && bun install && bun run build"
build-search-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: build
script:
only:
- merge_requests
- master
before_script:
- "pip install pipenv"
script:
- "cd dbrepo-search-service && pipenv install --system --deploy"
build-images:
image: docker.io/docker:${DOCKER_VERSION}-dind
stage: build
only:
- merge_requests
- master
before_script:
- "apk add --no-cache make"
- echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL
......@@ -214,6 +245,10 @@ build-images:
build-helm:
image: docker.io/docker:${DOCKER_VERSION}-dind
stage: build
only:
- tags
- merge_requests
- master
before_script:
- apk add --no-cache helm make
- echo "$CI_GPG_KEYRING" | base64 -d > ./secring.gpg
......@@ -221,30 +256,12 @@ build-helm:
script:
- make build-helm
deploy-staging:
image: docker.io/alpine:${ALPINE_VERSION}
stage: deploy
variables:
KUBECONFIG: ./kubecfg
environment:
name: staging/datalab
url: ${CI_ENV_STAGING_URL}
before_script:
- apk add --no-cache helm kubectl make
- echo ${CI_K8S_CONFIG} | base64 -d > ./kubecfg
- make build-helm
- helm -n ${CI_ENV_STAGING_NAMESPACE} uninstall ${CI_ENV_STAGING_RELEASE_NAME} --ignore-not-found
- kubectl -n ${CI_ENV_STAGING_NAMESPACE} delete pvc --all
script:
- helm -n ${CI_ENV_STAGING_NAMESPACE} upgrade --install ${CI_ENV_STAGING_RELEASE_NAME} ./build/dbrepo-${CHART_VERSION}.tgz --create-namespace -f ./.gitlab/agents/dev/values.yaml
only:
refs:
- dev
- master
test-metadata-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: test
only:
- merge_requests
- master
needs:
- build-metadata-service
dependencies:
......@@ -265,6 +282,9 @@ test-metadata-service:
test-data-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: test
only:
- merge_requests
- master
needs:
- build-data-service
dependencies:
......@@ -283,16 +303,12 @@ test-data-service:
junit: ./dbrepo-data-service/rest-service/target/surefire-reports/TEST-*.xml
coverage: '/Total.*?([0-9]{1,3})%/'
test-upload-service:
image: maven:3-openjdk-${JAVA_VERSION}
stage: test
script:
- "mvn -f ./dbrepo-metadata-service/pom.xml clean install $MAVEN_OPTS -DskipTests"
- "mvn -f ./dbrepo-upload-service/pom.xml clean test $MAVEN_OPTS"
test-analyse-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
only:
- merge_requests
- master
variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
needs:
......@@ -302,21 +318,24 @@ test-analyse-service:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./dbrepo-analyse-service/ && coverage run -m pytest tests/test_determine_dt.py tests/test_determine_pk.py tests/test_s3_client.py --junitxml=report.xml && coverage html --omit="tests/*" && coverage report --omit="tests/*" > ./coverage.txt
- cd ./dbrepo-analyse-service/ && coverage run --rcfile=.coveragerc -m pytest tests/test_determine_dt.py tests/test_determine_pk.py tests/test_s3_client.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./dbrepo-analyse-service/report.xml
- ./dbrepo-analyse-service/coverage.xml
- ./dbrepo-analyse-service/coverage.txt
expire_in: 1 days
reports:
junit: ./dbrepo-analyse-service/report.xml
junit: ./dbrepo-analyse-service/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-auth-service-init:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
only:
- merge_requests
- master
variables:
PIPENV_PIPFILE: "./dbrepo-auth-service/init/Pipfile"
before_script:
......@@ -324,21 +343,24 @@ test-auth-service-init:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
script:
- cd ./dbrepo-auth-service/init/ && coverage run -m pytest tests/test_unit_app.py --junitxml=report.xml && coverage html --omit="tests/*" && coverage report --omit="tests/*" > ./coverage.txt
- cd ./dbrepo-auth-service/init/ && coverage run --rcfile=.coveragerc -m pytest tests/test_unit_app.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./dbrepo-auth-service/init/report.xml
- ./dbrepo-auth-service/init/coverage.xml
- ./dbrepo-auth-service/init/coverage.txt
expire_in: 1 days
reports:
junit: ./dbrepo-auth-service/init/report.xml
junit: ./dbrepo-auth-service/init/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-search-service:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
only:
- merge_requests
- master
variables:
PIPENV_PIPFILE: "./dbrepo-search-service/Pipfile"
needs:
......@@ -355,21 +377,24 @@ test-search-service:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./dbrepo-search-service/ && coverage run -m pytest tests/test_app.py tests/test_jwt.py tests/test_opensearch_client.py tests/test_keycloak_client.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt
- cd ./dbrepo-search-service/ && coverage run --rcfile=.coveragerc -m pytest tests/test_app.py tests/test_jwt.py tests/test_opensearch_client.py tests/test_keycloak_client.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./dbrepo-search-service/report.xml
- ./dbrepo-search-service/coverage.xml
- ./dbrepo-search-service/coverage.txt
expire_in: 1 days
reports:
junit: ./dbrepo-search-service/report.xml
junit: ./dbrepo-search-service/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-search-service-init:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
only:
- merge_requests
- master
variables:
PIPENV_PIPFILE: "./dbrepo-search-service/init/Pipfile"
needs:
......@@ -379,21 +404,24 @@ test-search-service-init:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./dbrepo-search-service/init/ && coverage run -m pytest tests/test_app.py tests/test_keycloak_client.py tests/test_opensearch_client.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt
- cd ./dbrepo-search-service/init/ && coverage run --rcfile=.coveragerc -m pytest tests/test_app.py tests/test_keycloak_client.py tests/test_opensearch_client.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./dbrepo-search-service/report.xml
- ./dbrepo-search-service/coverage.xml
- ./dbrepo-search-service/coverage.txt
expire_in: 1 days
reports:
junit: ./dbrepo-search-service/report.xml
junit: ./dbrepo-search-service/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-lib:
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
stage: test
only:
- merge_requests
- master
variables:
PIPENV_PIPFILE: "./lib/python/Pipfile"
needs:
......@@ -403,21 +431,24 @@ test-lib:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- cd ./lib/python/ && coverage run -m pytest tests/test_unit_analyse.py tests/test_unit_container.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py tests/test_unit_rest_client.py --junitxml=report.xml && coverage html --omit="tests/*" && coverage report --omit="tests/*" > ./coverage.txt
- cd ./lib/python/ && coverage run --rcfile=.coveragerc -m pytest tests/test_unit_container.py tests/test_unit_messages.py tests/test_unit_image.py tests/test_unit_concept.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py tests/test_unit_unit.py && coverage html && coverage xml && coverage report > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./lib/python/report.xml
- ./lib/python/coverage.xml
- ./lib/python/coverage.txt
expire_in: 1 days
reports:
junit: ./lib/python/report.xml
junit: ./lib/python/coverage.xml
coverage: '/TOTAL.*?([0-9]{1,3})%/'
test-ui:
stage: test
image: docker.io/docker:${DOCKER_VERSION}-dind
stage: test
only:
- merge_requests
- master
needs:
- build-ui
dependencies:
......@@ -433,13 +464,18 @@ test-ui:
release-images:
stage: release
image: docker:${DOCKER_VERSION}-dind
only:
- dev
- tags
dependencies:
- test-metadata-service
- test-data-service
- test-analyse-service
only:
refs:
- /^release-.*/
- test-auth-service-init
- test-data-service
- test-lib
- test-metadata-service
- test-search-service
- test-search-service-init
- test-ui
before_script:
- "docker logout ${CI_REGISTRY_URL}"
- "echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY_URL}"
......@@ -454,9 +490,7 @@ release-helm:
stage: release
image: docker:${DOCKER_VERSION}-dind
only:
refs:
- /^release-.*/
when: manual
- tags
needs:
- build-helm
dependencies:
......@@ -470,6 +504,8 @@ release-helm:
- "mkdir -p ~/.gnupg"
- echo "$CI_GPG_KEYRING" | base64 -d > ~/.gnupg/secring.gpg
- echo "$CI_GPG_KEYRING2" | base64 -d > ~/.gnupg/pubring.gpg
- helm registry login --username "${CI_REGISTRY_USER}" --password "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY_URL}"
- helm registry login --username "${CI_REGISTRY2_USER}" --password "${CI_REGISTRY2_PASSWORD}" "${CI_REGISTRY2_URL}"
- make build-helm
- "helm package ./helm/dbrepo --sign --key 'Martin Weise' --keyring ~/.gnupg/secring.gpg --destination ./build"
- "helm plugin install https://github.com/sigstore/helm-sigstore"
......@@ -481,8 +517,7 @@ release-docs:
stage: release
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
only:
refs:
- /^release-.*/
- tags
before_script:
- "apk add --no-cache alpine-sdk bash sed wget openssh jq curl"
- "pip install pipenv"
......@@ -514,10 +549,8 @@ release-docs:
release-libs:
stage: release
image: docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
when: manual
only:
refs:
- /^release-.*/
- tags
variables:
PIPENV_PIPFILE: "./dbrepo-analyse-service/Pipfile"
before_script:
......@@ -532,8 +565,7 @@ verify-install-script:
image: docker.io/docker:${DOCKER_VERSION}-dind
stage: verify
only:
refs:
- /^release-.*/
- tags
variables:
SKIP_CHECKS: 1
before_script:
......@@ -545,25 +577,37 @@ verify-dist:
image: docker.io/alpine:${ALPINE_VERSION}
stage: verify
only:
refs:
- /^release-.*/
- tags
before_script:
- "apk add curl"
script:
- "curl -v --output /dev/null --fail https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${APP_VERSION}/dist.tar.gz"
deploy-staging:
image: docker.io/alpine:${ALPINE_VERSION}
stage: deploy
only:
- dev
variables:
KUBECONFIG: ./kubecfg
environment:
name: staging/datalab
url: ${CI_ENV_STAGING_URL}
before_script:
- apk add --no-cache helm kubectl make
- echo ${CI_K8S_CONFIG} | base64 -d > ./kubecfg
- make build-helm
- helm -n ${CI_ENV_STAGING_NAMESPACE} uninstall ${CI_ENV_STAGING_RELEASE_NAME} --ignore-not-found
- kubectl -n ${CI_ENV_STAGING_NAMESPACE} delete pvc --all
- echo "${CI_K8S_VALUES}" | base64 -d > ./values.yaml
script:
- helm -n ${CI_ENV_STAGING_NAMESPACE} upgrade --install ${CI_ENV_STAGING_RELEASE_NAME} ./build/dbrepo-${CHART_VERSION}.tgz --create-namespace -f ./values.yaml
scan-sonarqube:
image: sonarsource/sonar-scanner-cli:${SONARQUBE_VERSION}
stage: scan
only:
refs:
- master
needs:
- build-data-service
- build-metadata-service
dependencies:
- build-data-service
- build-metadata-service
- master
script:
- 'sonar-scanner -Dsonar.token="${CI_SONAR_TOKEN}"'
allow_failure: true
......
.gitlab/agents/dev/config.yaml deleted 100644 → 0
View file @ 79a3df8a
.gitlab/agents/dev/values.yaml deleted 100644 → 0
View file @ 79a3df8a
hostname: s155.datalab.tuwien.ac.at
gateway: https://s155.datalab.tuwien.ac.at
metadatadb:
enabled: true
rootUser:
user: root
password: da19c7cf5c0deba7bd47c174a0eb273b
galera:
mariabackup:
user: mariabackup
password: 9e447eeaf3e4b6aa26ea01582f0e8a54
persistence:
enabled: true
authservice:
enabled: true
auth:
adminUser: admin
adminPassword: ea72038fa14b968fc0ed09e182ecf624
postgresql:
auth:
postgresPassword: 129d5b888b8df271fa482da39f15c513
jwt:
pubkey: "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB"
client:
id: dbrepo-client
secret: MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG
setupJob:
image:
name: registry.datalab.tuwien.ac.at/dbrepo/auth-service-init:1.6.3
persistence:
enabled: true
brokerservice:
enabled: true
ldap:
bindpw: b8534187c9adf9618e7bd1c79c7f4639
identityservice:
enabled: true
global:
adminUser: admin
adminPassword: b8534187c9adf9618e7bd1c79c7f4639
users: admin
userPasswords: ea72038fa14b968fc0ed09e182ecf624
datadb:
enabled: true
rootUser:
user: root
password: fdf8578499b2083eb3aa03a861ac7912
galera:
mariabackup:
user: mariabackup
password: ef60e32e3217525474635cd28422c829
replicaCount: 3
persistence:
enabled: true
searchdb:
enabled: true
security:
enabled: false
extraEnvs:
- name: DISABLE_INSTALL_DEMO_CONFIG
value: "true"
persistence:
enabled: true
analyseservice:
enabled: true
image:
name: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.6.3
metadataservice:
enabled: true
image:
name: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.6.3
admin:
email: noreply@example.com
deletedRecord: permanent
repositoryName: Database Repository
granularity: YYYY-MM-DDThh:mm:ssZ
datacite:
enabled: false
url: https://api.datacite.org
prefix: ""
username: ""
password: ""
dataservice:
enabled: true
image:
name: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.6.3
rabbitmq:
consumer:
username: admin
password: ea72038fa14b968fc0ed09e182ecf624
s3:
auth:
username: a45e7a77607a8906e92237f00ea72f58
password: e2c4303dcbfd3a2c606fe30d19fcb82b
filePath: /s3
searchservice:
enabled: true
image:
name: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.6.3
init:
image:
name: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.6.3
storageservice:
enabled: true
init:
image:
name: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.6.3
uploadservice:
enabled: true
dashboardservice:
enabled: true
metricdb:
enabled: true
server:
rbac:
create: false
ui:
enabled: true
image:
name: registry.datalab.tuwien.ac.at/dbrepo/ui:1.6.3
public:
api:
client: https://s155.datalab.tuwien.ac.at
server: https://s155.datalab.tuwien.ac.at
title: "Database Repository"
logo: "https://ec.tuwien.ac.at/~weise/images/DS_white_hiRes.png"
icon: "https://ec.tuwien.ac.at/~weise/images/favicon.ico"
touch: "https://ec.tuwien.ac.at/~weise/images/DS-icon_white_hiRes.png"
broker:
host: s155.datalab.tuwien.ac.at
extra: "128.130.0.0/15"
database:
extra: "128.130.0.0/15"
pid:
default:
publisher: "TU Wien"
doi:
enabled: false
endpoint: https://doi.org
extraVolumes: [ ]
# - name: images-map
# configMap:
# name: ui-config
extraVolumeMounts: [ ]
# - name: images-map
# mountPath: /static/logo.svg
# subPath: logo.svg
gatewayservice:
enabled: true
ingress:
enabled: true
className: nginx
tls:
enabled: true
secretName: ingress-cert
annotations:
cert-manager.io/cluster-issuer: letsencrypt-cluster-issuer
# nginx.ingress.kubernetes.io/whitelist-source-range: 128.130.0.0/15
Makefile
View file @ 841d4894
.PHONY: all
APP_VERSION ?= 1.6.5
CHART_VERSION ?= 1.6.5
APP_VERSION ?= 1.7.0
CHART_VERSION ?= 1.7.0
REPOSITORY_URL ?= registry.datalab.tuwien.ac.at/dbrepo
.PHONY: all
......
README.md
View file @ 841d4894
......@@ -14,7 +14,7 @@ If you have [Docker](https://docs.docker.com/engine/install/) already installed
with:
```bash
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.6/install.sh | bash
curl -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.7/install.sh | bash
```
## Documentation
......
dbrepo-analyse-service/.coveragerc 0 → 100644
View file @ 841d4894
[report]
format = text
omit =
# omit tests
./tests/*
# omit ext lib
./omlib/*
[html]
directory = htmlcov
\ No newline at end of file
dbrepo-analyse-service/Pipfile
View file @ 841d4894
......@@ -21,7 +21,7 @@ numpy = "*"
pandas = "*"
minio = "*"
pydantic = "*"
dbrepo = {path = "./lib/dbrepo-1.6.5.tar.gz"}
dbrepo = {path = "./lib/dbrepo-1.7.0.tar.gz"}
opensearch-py = "*"
[dev-packages]
......
dbrepo-analyse-service/Pipfile.lock
View file @ 841d4894
This diff is collapsed.