Skip to content
Snippets Groups Projects
Commit bd5de40e authored by Martin Weise's avatar Martin Weise
Browse files

Merge branch 'hotfix-foreign-key' into 'dev' 

Hotfix foreign key

See merge request !274
parents 52b0e1f6 a25cc712
No related branches found
No related tags found
2 merge requests!275Dev,!274Hotfix foreign key
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 7929 additions and 2096 deletions
docker-compose.prod.yml → .docker/docker-compose.yml
+ 25
− 19
View file @ bd5de40e
......@@ -14,7 +14,7 @@ services:
restart: "no"
container_name: dbrepo-metadata-db
hostname: metadata-db
image: docker.io/dbrepo/metadata-db:latest
image: docker.io/dbrepo/metadata-db:1.4.4
volumes:
- metadata-db-data:/bitnami/mariadb
- ./dist/2_setup-data.sql:/docker-entrypoint-initdb.d/2_setup-data.sql
......@@ -76,7 +76,7 @@ services:
restart: "no"
container_name: dbrepo-auth-service
hostname: auth-service
image: docker.io/dbrepo/auth-service:latest
image: docker.io/dbrepo/auth-service:1.4.4
healthcheck:
test: curl -sSL 'http://0.0.0.0:8080/realms/dbrepo' | grep "dbrepo" || exit 1
interval: 10s
......@@ -98,18 +98,19 @@ services:
restart: "no"
container_name: dbrepo-metadata-service
hostname: metadata-service
image: docker.io/dbrepo/metadata-service:latest
image: docker.io/dbrepo/metadata-service:1.4.4
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
ADMIN_MAIL: "${ADMIN_MAIL:-noreply@localhost}"
ADMIN_EMAIL: "${ADMIN_EMAIL:-noreply@localhost}"
ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
ANALYSE_SERVICE_ENDPOINT: "${ANALYSE_SERVICE_ENDPOINT:-http://gateway-service}"
AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-fda}
AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-fda}
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://gateway-service/api/auth}
BASE_URL: "${BASE_URL:-http://localhost}"
BROKER_EXCHANGE_NAME: ${BROKER_EXCHANGE_NAME:-dbrepo}
BROKER_QUEUE_NAME: ${BROKER_QUEUE_NAME:-dbrepo}
......@@ -121,10 +122,9 @@ services:
BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}"
DATA_SERVICE_ENDPOINT: ${DATA_SERVICE_ENDPOINT:-http://data-service:8080}
DELETED_RECORD: "${DELETED_RECORD:-persistent}"
GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
GRANULARITY: "${GRANULARITY:-YYYY-MM-DDThh:mm:ssZ}"
JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"
LOG_LEVEL: ${LOG_LEVEL:-info}
LOG_LEVEL: "${LOG_LEVEL:-info}"
METADATA_DB: "${METADATA_DB:-dbrepo}"
METADATA_HOST: "${METADATA_HOST:-metadata-db}"
METADATA_JDBC_EXTRA_ARGS: "${METADATA_JDBC_EXTRA_ARGS:-}"
......@@ -132,9 +132,9 @@ services:
METADATA_PASSWORD: "${METADATA_PASSWORD:-dbrepo}"
PID_BASE: ${PID_BASE:-http://localhost/pid/}
REPOSITORY_NAME: "${REPOSITORY_NAME:-Database Repository}"
SEARCH_SERVICE_ENDPOINT: "${SEARCH_SERVICE_ENDPOINT:-http://search-service:8080}"
SEARCH_SERVICE_ENDPOINT: "${SEARCH_SERVICE_ENDPOINT:-http://gateway-service}"
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
S3_ENDPOINT: "${S3_ENDPOINT:-http://gateway-service/api/storage}"
S3_EXPORT_BUCKET: "${S3_EXPORT_BUCKET:-dbrepo-download}"
S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
......@@ -160,7 +160,7 @@ services:
restart: "no"
container_name: dbrepo-analyse-service
hostname: analyse-service
image: docker.io/dbrepo/analyse-service:latest
image: docker.io/dbrepo/analyse-service:1.4.4
environment:
ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
......@@ -211,7 +211,7 @@ services:
restart: "no"
container_name: dbrepo-search-db
hostname: search-db
image: docker.io/dbrepo/search-db:latest
image: docker.io/dbrepo/search-db:1.4.4
healthcheck:
test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP
interval: 10s
......@@ -235,7 +235,7 @@ services:
restart: "no"
container_name: dbrepo-search-service
hostname: search-service
image: docker.io/dbrepo/search-service:latest
image: docker.io/dbrepo/search-service:1.4.4
environment:
ADMIN_PASSWORD: "${ADMIN_PASSWORD:-admin}"
ADMIN_USERNAME: "${ADMIN_USERNAME:-admin}"
......@@ -253,11 +253,12 @@ services:
restart: "no"
container_name: dbrepo-data-db-sidecar
hostname: data-db-sidecar
image: docker.io/dbrepo/data-db-sidecar:latest
image: docker.io/dbrepo/data-db-sidecar:1.4.4
environment:
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
S3_EXPORT_BUCKET: "${S3_EXPORT_BUCKET:-dbrepo-download}"
S3_FILE_PATH: "${S3_FILE_PATH:-/tmp}"
S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
volumes:
......@@ -274,7 +275,7 @@ services:
restart: "no"
container_name: dbrepo-ui
hostname: ui
image: docker.io/dbrepo/ui:latest
image: docker.io/dbrepo/ui:1.4.4
depends_on:
dbrepo-search-service:
condition: service_started
......@@ -318,7 +319,7 @@ services:
restart: "no"
container_name: dbrepo-search-service-init
hostname: search-service-init
image: docker.io/dbrepo/search-service-init:latest
image: docker.io/dbrepo/search-service-init:1.4.4
environment:
GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
......@@ -353,7 +354,7 @@ services:
restart: "no"
container_name: dbrepo-storage-service-init
hostname: storage-service-init
image: docker.io/dbrepo/storage-service-init:latest
image: docker.io/dbrepo/storage-service-init:1.4.4
environment:
SEAWEEDFS_ENDPOINT: "${STORAGE_SEAWEEDFS_ENDPOINT:-storage-service:9333}"
depends_on:
......@@ -390,7 +391,7 @@ services:
restart: "no"
container_name: dbrepo-data-service
hostname: data-service
image: docker.io/dbrepo/data-service:latest
image: docker.io/dbrepo/data-service:1.4.4
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
......@@ -411,7 +412,7 @@ services:
BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}"
CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000}
EXCHANGE_NAME: ${EXCHANGE_NAME:-dbrepo}
GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://gateway-service}
GRANT_DEFAULT_READ: "${GRANT_DEFAULT_READ:-SELECT}"
GRANT_DEFAULT_WRITE: "${GRANT_DEFAULT_WRITE:-SELECT, CREATE, CREATE VIEW, CREATE ROUTINE, CREATE TEMPORARY TABLES, LOCK TABLES, INDEX, TRIGGER, INSERT, UPDATE, DELETE}"
JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"
......@@ -421,7 +422,12 @@ services:
QUEUE_NAME: ${QUEUE_NAME:-dbrepo}
REQUEUE_REJECTED: ${REQUEUE_REJECTED:-false}
ROUTING_KEY: "${ROUTING_KEY:-dbrepo.#}"
STORAGE_SERVICE_ENDPOINT: ${BROKER_SERVICE_ENDPOINT:-http://storage-service:9000}
S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
S3_EXPORT_BUCKET: "${S3_EXPORT_BUCKET:-dbrepo-download}"
S3_FILE_PATH: "${S3_FILE_PATH:-/tmp}"
S3_IMPORT_BUCKET: "${S3_IMPORT_BUCKET:-dbrepo-upload}"
S3_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY:-seaweedfsadmin}"
healthcheck:
test: wget -qO- localhost:8080/actuator/health/readiness | grep -q "UP" || exit 1
interval: 10s
......
.docs/.swagger/api.yaml
+ 7805
− 2010
View file @ bd5de40e
This diff is collapsed.
.docs/.swagger/openapi-merge.json 0 → 100644
+ 23
− 0
View file @ bd5de40e
{
"inputs": [
{
"inputFile": "./api.base.yaml"
},
{
"inputFile": "./api-analyse.yaml"
},
{
"inputFile": "./api-data.yaml"
},
{
"inputFile": "./api-metadata.yaml"
},
{
"inputFile": "./api-search.yaml"
},
{
"inputFile": "./api-sidecar.yaml"
}
],
"output": "./api.yaml"
}
.docs/.swagger/swagger-ui.html
+ 1
− 1
View file @ bd5de40e
......@@ -7,7 +7,7 @@
<title>DBRepo REST API</title>
<link rel="stylesheet" href="https://unpkg.com/swagger-ui-dist@5.17.12/swagger-ui.css"/>
<link rel="stylesheet" href="./custom.css"/>
<link rel="icon" href="https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/master/.docs/images/signet_white.png" />
<link rel="icon" href="https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/master/.docs/images/logos/favicon.png" />
</head>
<body>
<div class="swagger-ui">
......
.docs/api/analyse-service.md
+ 7
− 7
View file @ bd5de40e
......@@ -11,19 +11,19 @@ author: Martin Weise
* Ports: 5000/tcp
* Prometheus: `http://<hostname>:5000/metrics`
* Health: `http://<hostname>:5000/health`
* Swagger UI: `http://<hostname>:5000/swagger-ui/` <a href="../swagger/analyse" target="_blank">:fontawesome-solid-square-up-right: view online</a>
* Swagger UI: `http://<hostname>:5000/swagger-ui/` <a href="./swagger/analyse" target="_blank">:fontawesome-solid-square-up-right: view online</a>
## Overview
It suggests data types for the [User Interface](../system-other-ui) when creating a table from a
It suggests data types for the [User Interface](./system-other-ui) when creating a table from a
*comma separated values* (CSV) -file. It recommends enumerations for columns and returns e.g. a list of potential
primary key candidates. The researcher is able to confirm these suggestions manually. Moreover, the Analyse Service
determines basic statistical properties of numerical columns.
### Analysis
After [uploading](../system-services-storage/#buckets) the CSV-file into the `dbrepo-upload` bucket of
the [Storage Service](../system-services-storage), analysis for data types and primary keys follows the flow:
After [uploading](./system-services-storage/#buckets) the CSV-file into the `dbrepo-upload` bucket of
the [Storage Service](./system-services-storage), analysis for data types and primary keys follows the flow:
1. Retrieve the CSV-file from the `dbrepo-upload` bucket of the Storage Service as data stream (=nothing is stored in
the service) with the [`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) client.
......@@ -36,16 +36,16 @@ the [Storage Service](../system-services-storage), analysis for data types and p
### Examples
See the [usage page](../usage-analyse/) for examples.
See the [usage page](./usage-analyse/) for examples.
## Limitations
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../contact) with us, we happily answer requests for collaboration with attached CV and your programming
in [contact](./contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
## Security
1. Credentials for the [Storage Service](../system-services-storage) are stored in plaintext environment variables.
1. Credentials for the [Storage Service](./system-services-storage) are stored in plaintext environment variables.
.docs/api/data-service.md
+ 2
− 2
View file @ bd5de40e
......@@ -14,7 +14,7 @@ author: Martin Weise
- Readiness: `http://<hostname>:9093/actuator/health/readiness`
- Liveness: `http://<hostname>:9093/actuator/health/liveness`
* Prometheus: `http://<hostname>:9093/actuator/prometheus`
* Swagger UI: `http://<hostname>:9093/swagger-ui/index.html` <a href="../swagger/data" target="_blank">:fontawesome-solid-square-up-right: view online</a>
* Swagger UI: `http://<hostname>:9093/swagger-ui/index.html` <a href="./swagger/data" target="_blank">:fontawesome-solid-square-up-right: view online</a>
## Overview
......@@ -27,7 +27,7 @@ Data Service up.
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../contact) with us, we happily answer requests for collaboration with attached CV and your programming
in [contact](./contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
## Security
......
.docs/api/gateway-service.md
+ 2
− 2
View file @ bd5de40e
......@@ -41,7 +41,7 @@ If your TLS private key as a password, you need to specify it in the `dbrepo.con
### User Interface
To serve the [User Interface](../system-other-ui/) under different port than `80`, change the port mapping in
To serve the [User Interface](./system-other-ui/) under different port than `80`, change the port mapping in
the `docker-compose.yml` to e.g. port `8000`:
```yaml title="docker-compose.yml"
......@@ -61,7 +61,7 @@ services:
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../contact) with us, we happily answer requests for collaboration with attached CV and your programming
in [contact](./contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
......
.docs/api/index.md
+ 12
− 12
View file @ bd5de40e
......@@ -5,7 +5,7 @@ author: Martin Weise
# Overview
We developed a Python Library for communicating with DBRepo from e.g. Jupyter Notebooks. See
the [Python Library](../usage-python) page for more details.
the [Python Library](./usage-python) page for more details.
We give usage examples of the most important use-cases we identified.
......@@ -30,7 +30,7 @@ A user wants to create an account in DBRepo.
button and provide a valid work e-mail address :material-numeric-1-circle-outline: and a username (in lowercase
alphanumeric characters) :material-numeric-2-circle-outline:. Choose a secure password in field
:material-numeric-3-circle-outline: and repeat it in field :material-numeric-4-circle-outline:. Click "SUBMIT" and
the system creates a user account in Figure 1 with the [default roles](../system-services-authentication/#roles)
the system creates a user account in Figure 1 with the [default roles](./system-services-authentication/#roles)
that your administrator has assigned.
<figure markdown>
......@@ -425,7 +425,7 @@ A user wants to import a database dump in `.sql` (or in `.sql.gz`) format into D
Setup a new connection in the MySQL Workbench (c.f. Figure 14) by clicking the small
":material-plus-circle-outline:" button :material-numeric-1-circle-outline: to open the dialog. In the opened dialog
fill out the connection parameters (for local deployments the hostname is `127.0.0.1` and port `3307` for the
[Data Database](../system-databases-data/) :material-numeric-2-circle-outline:.
[Data Database](./system-databases-data/) :material-numeric-2-circle-outline:.
The default credentials are username `root` and password `dbrepo`, type the password in
:material-numeric-3-circle-outline: and click the "OK" button. Then finish the setup of the new connection by
......@@ -470,8 +470,8 @@ A user wants to import a database dump in `.sql` (or in `.sql.gz`) format into D
gunzip < dump.sql.gz | mysql -H127.0.0.1 -p3307 -uUSERNAME -pYOURPASSWORD db_name
```
The [Metadata Service](../system-services-metadata) periodically (by default configuration every 60 seconds) checks
and adds missing tables and views to the [Metadata Database](../system-databases-metadata), the database dump
The [Metadata Service](./system-services-metadata) periodically (by default configuration every 60 seconds) checks
and adds missing tables and views to the [Metadata Database](./system-databases-metadata), the database dump
will be visible afterwards. Currently, date formats for columns with time types (e.g. `DATE`, `TIMESTAMP`) are
assumed to match the first date format found for the database image. This may need to be manually specified by the
administrator.
......@@ -479,7 +479,7 @@ A user wants to import a database dump in `.sql` (or in `.sql.gz`) format into D
!!! example "Specifying a custom date format"
In case the pre-defined date formats are not matching the found date format in the database dump, the system
administrator needs to add it manually in the [Metadata Database](../system-databases-metadata).
administrator needs to add it manually in the [Metadata Database](./system-databases-metadata).
```sql
INSERT INTO `mdb_images_date` (`iid`, `database_format`, `unix_format`, `example`, `has_time`)
......@@ -658,7 +658,7 @@ A user wants to create a subset and export it as csv file.
```
Afterwards, you can see the subset in the UI with subset id `@subsetId` and persist it there. Only the administrator
can persist the subset in the [Data Database](../system-databases-data) through JDBC by setting the `persisted`
can persist the subset in the [Data Database](./system-databases-data) through JDBC by setting the `persisted`
column to `true` in the `qs_queries` table.
=== "Python"
......@@ -732,7 +732,7 @@ A user wants to assign a persistent identifier to a database owned by them.
<figcaption>Figure 21: Open the get persisent identifier form.</figcaption>
</figure>
First, provide information on the dataset creator(s). Since the [Metadata Service](../system-services-metadata)
First, provide information on the dataset creator(s). Since the [Metadata Service](./system-services-metadata)
automatically resolves external PIDs, the easiest way is to provide the correct mandatory data is by filling the
name identifier :material-numeric-1-circle-outline:. The creator type :material-numeric-2-circle-outline:
denotes either a natural person or organization. Optionally fill out the given
......@@ -779,7 +779,7 @@ A user wants to assign a persistent identifier to a database owned by them.
<figcaption>Figure 25: Related identifiers, license and language of the identifier.</figcaption>
</figure>
Optionally add funding information, again the [Metadata Service](../system-services-metadata)
Optionally add funding information, again the [Metadata Service](./system-services-metadata)
automatically resolves external PIDs, the easiest way is to provide the correct mandatory data is by filling the
funder identifier :material-numeric-1-circle-outline: that attempts to get the funder
name :material-numeric-2-circle-outline:. If you provide an award number :material-numeric-3-circle-outline: and/or
......@@ -817,11 +817,11 @@ A user wants to assign a persistent identifier to a database owned by them.
!!! warning
Creating a PID directly in the [Metadata Database](../system-databases-metadata) is not recommended! It bypasses
Creating a PID directly in the [Metadata Database](./system-databases-metadata) is not recommended! It bypasses
validation and creation of external PIDs (e.g. DOI) and may lead to inconstistent data locally compared to
external systems (e.g. DataCite Fabrica).
Create a local PID directly in the [Metadata Database](../system-databases-metadata) by filling the tables in this
Create a local PID directly in the [Metadata Database](./system-databases-metadata) by filling the tables in this
order (they have foreign key dependencies).
1. `mdb_identifiers` ... identifier core information
......@@ -928,7 +928,7 @@ A user wants a public database to be private and only give specific users access
=== "JDBC"
To change the visibility of a database as administrator with direct JDBC access to
the [Metadata Database](../system-databases-metadata), change the visibility directly by executing the SQL-query
the [Metadata Database](./system-databases-metadata), change the visibility directly by executing the SQL-query
in the `fda` schema:
```sql
......
.docs/api/metadata-service.md
+ 10
− 10
View file @ bd5de40e
......@@ -14,7 +14,7 @@ author: Martin Weise
- Readiness: `http://<hostname>:9099/actuator/health/readiness`
- Liveness: `http://<hostname>:9099/actuator/health/liveness`
* Prometheus: `http://<hostname>:9099/actuator/prometheus`
* Swagger UI: `http://<hostname>:9099/swagger-ui/index.html` <a href="../swagger/metadata" target="_blank">:fontawesome-solid-square-up-right: view online</a>
* Swagger UI: `http://<hostname>:9099/swagger-ui/index.html` <a href="./swagger/metadata" target="_blank">:fontawesome-solid-square-up-right: view online</a>
## Overview
......@@ -75,23 +75,23 @@ Executing SQL queries through the Query Endpoint must fulfill some restrictions:
### Semantics
The service provides metadata to the table columns in the [Metadata Database](../system-databases-metadata) from
The service provides metadata to the table columns in the [Metadata Database](./system-databases-metadata) from
registered ontologies like Wikidata [`wd:`](https://wikidata.org), Ontology of Units of
Measurement [`om2:`](https://www.ontology-of-units-of-measure.org/resource/om-2), Friend of a
Friend [`foaf:`](http://xmlns.com/foaf/0.1/), the [`prov:`](http://www.w3.org/ns/prov#) namespace, etc.
### Tables
The service manages tables in the [Data Database](../system-databases-data) and manages the metadata of these tables
in the [Metadata Database](../system-databases-metadata). Any tables that are created outside of DBRepo (e.g. directly via the JDBC API) are
The service manages tables in the [Data Database](./system-databases-data) and manages the metadata of these tables
in the [Metadata Database](./system-databases-metadata). Any tables that are created outside of DBRepo (e.g. directly via the JDBC API) are
periodically fetched by this service (based on the `OBTAIN_METADATA_RATE` environment variable, default interval is 60
seconds).
### Users
The service manages users in the [Data Database](../system-databases-data)
and [Metadata Database](../system-databases-metadata), as well as in the [Broker Service](../system-services-broker)
and the [Authentication Service](../system-services-authentication).
The service manages users in the [Data Database](./system-databases-data)
and [Metadata Database](./system-databases-metadata), as well as in the [Broker Service](./system-services-broker)
and the [Authentication Service](./system-services-authentication).
The default configuration grants the users only very basic permissions on the databases:
......@@ -123,8 +123,8 @@ A list of all grants is available in the MariaDB documentation for [`GRANT`](htt
### Views
The service manages views in the [Data Database](../system-databases-data)
and [Metadata Database](../system-databases-metadata). Any views that are created outside of DBRepo (e.g. directly via
The service manages views in the [Data Database](./system-databases-data)
and [Metadata Database](./system-databases-metadata). Any views that are created outside of DBRepo (e.g. directly via
the JDBC API) are periodically fetched by this service (based on the `OBTAIN_METADATA_RATE` environment variable,
default interval is 60 seconds).
......@@ -136,7 +136,7 @@ default interval is 60 seconds).
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../contact) with us, we happily answer requests for collaboration with attached CV and your programming
in [contact](./contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
## Security
......
.docs/api/open-api.md
+ 9
− 3
View file @ bd5de40e
......@@ -2,6 +2,12 @@
author: Martin Weise
---
All services are documented using the
[![OpenAPI 3.0](https://img.shields.io/badge/OpenAPI-3.0-leaf?logo=swagger&logoColor=white)](https://www.openapis.org/){ tabindex=-1 }
documentation standard.
![OpenAPI 3.1](https://img.shields.io/badge/OpenAPI-3.1-leaf)
## tl;dr
[:simple-swagger: &nbsp;View Swagger-UI](../../swagger/){ .md-button .md-button--primary tabindex=-1 }
## Overview
All services are documented using the [OpenAPI 3.1](https://www.openapis.org/) documentation standard.
\ No newline at end of file
.docs/api/storage-service.md
+ 1
− 1
View file @ bd5de40e
......@@ -36,7 +36,7 @@ The default configuration creates two buckets `dbrepo-upload`, `dbrepo-download`
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../contact) with us, we happily answer requests for collaboration with attached CV and your programming
in [contact](./contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
## Security
......
.docs/deployment-docker-compose.md
+ 4
− 4
View file @ bd5de40e
......@@ -53,7 +53,7 @@ technologies. The conceptualized microservices operate the basic database operat
### Notes
Please note that we only save the state of the databases as well as the [Broker Service](../system-services-broker)
Please note that we only save the state of the databases as well as the [Broker Service](./system-services-broker)
since RabbitMQ maintains state inside the container.
## Deployment
......@@ -147,8 +147,8 @@ Please be warned that the default configuration is not intended for public deplo
running system within minutes to play around within the system and explore features. It is strongly advised to change
the default `.env` environment variables.
Next, create a [user account](../usage-overview/#create-user-account) and
then [create a database](../usage-overview/#create-database) to [import a dataset](../usage-overview/#import-dataset).
Next, create a [user account](./usage-overview/#create-user-account) and
then [create a database](./usage-overview/#create-database) to [import a dataset](./usage-overview/#import-dataset).
## Security
......@@ -193,4 +193,4 @@ then [create a database](../usage-overview/#create-database) to [import a datase
!!! info "Alternative Deployments"
Alternatively, you can also deploy DBRepo with [Helm](../deployment-helm/) in your virtual machine instead.
Alternatively, you can also deploy DBRepo with [Helm](./deployment-helm/) in your virtual machine instead.
.docs/deployment-helm.md
+ 3
− 3
View file @ bd5de40e
......@@ -32,12 +32,12 @@ about values, etc.
## Limitations
1. MariaDB Galera does not (yet) support XA-transactions required by the authentication service (=Keycloak). Therefore
only a single MariaDB pod can be deployed at once for the [auth database](../system-databases-authentication).
only a single MariaDB pod can be deployed at once for the [auth database](./system-databases-authentication).
2. The entire Helm deployment is rootless (=`runAsNonRoot=true`) except for
the [Storage Service](../system-services-storage/) which still requires a root user.
the [Storage Service](./system-services-storage/) which still requires a root user.
!!! question "Do you miss functionality? Do these limitations affect you?"
We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get
in [contact](../contact) with us, we happily answer requests for collaboration with attached CV and your programming
in [contact](./contact) with us, we happily answer requests for collaboration with attached CV and your programming
experience!
.docs/docker/_header.md
+ 3
− 3
View file @ bd5de40e
......@@ -10,7 +10,7 @@
# Supported tags
* [`1.4.x`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-__APP_VERSION__/dbrepo-DIR/Dockerfile/)
* [`1.4.3`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.3/dbrepo-DIR/Dockerfile/)
* [`latest`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-latest/dbrepo-DIR/Dockerfile/)
# Non-supported tags
......@@ -29,8 +29,8 @@
* **Source of this description:**
[docs repo's `.docs/docker` directory](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tree/release-__APP_VERSION__/.docs/docker)
([history](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/commits/release-__APP_VERSION__/.docs/docker))
[docs repo's `.docs/docker` directory](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tree/release-1.4.3/.docs/docker)
([history](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/commits/release-1.4.3/.docs/docker))
# What is DBRepo?
......
.docs/publications.md
+ 3
− 3
View file @ bd5de40e
......@@ -14,14 +14,14 @@ hide:
Semantic Digital Repository for Relational Databases. *International Journal of Digital Curation*, 17(1), 11.
DOI: [10.2218/ijdc.v17i1.825](https://doi.org/10.2218/ijdc.v17i1.825)<br />
[[BibTeX](../papers/weise2022dbrepo.bib)] &nbsp; [[RIS](../papers/weise2022dbrepo.ris)] &nbsp; [[RDF](../papers/weise2022dbrepo.rdf)] &nbsp; [[EndNote](../papers/weise2022dbrepo.xml)]
[[BibTeX](./papers/weise2022dbrepo.bib)] &nbsp; [[RIS](./papers/weise2022dbrepo.ris)] &nbsp; [[RDF](./papers/weise2022dbrepo.rdf)] &nbsp; [[EndNote](./papers/weise2022dbrepo.xml)]
## Logos
DBRepo logo in various formats:
* PNG: [bigger](../images/logo/logo.png) ([smaller](../images/logo/favicon.png))
* SVG: [bigger](../images/logo/logo.svg) ([smaller](../images/logo/favicon.svg))
* PNG: [bigger](./images/logo/logo.png) ([smaller](./images/logo/favicon.png))
* SVG: [bigger](./images/logo/logo.svg) ([smaller](./images/logo/favicon.svg))
## Refereed
......
.docs/usage-storage.md
+ 1
− 1
View file @ bd5de40e
......@@ -53,7 +53,7 @@ $ aws --endpoint-url http://localhost:9000 \
## Other
Alternatively, you can use the middleware of the [User Interface](../system-other-ui/) to upload files.
Alternatively, you can use the middleware of the [User Interface](./system-other-ui/) to upload files.
Alternatively, you can use a S3-compatible client:
......
.docs/usage-upload.md
+ 1
− 1
View file @ bd5de40e
......@@ -11,7 +11,7 @@ We recommend using a TUS-compatible client:
* [tus-js-client](https://github.com/tus/tus-js-client) (JavaScript/Node.js)
* [tusd](https://github.com/tus/tusd) (Go)
Upload a file to the `dbrepo-upload` bucket in the [Storage Service](../system-services-storage/) using the Node.js
Upload a file to the `dbrepo-upload` bucket in the [Storage Service](./system-services-storage/) using the Node.js
middleware.
=== "Terminal"
......
.gitlab-ci.yml
+ 13
− 10
View file @ bd5de40e
......@@ -5,8 +5,8 @@ variables:
DOCKER_HOST: "unix:///var/run/dind/docker.sock"
TESTCONTAINERS_RYUK_DISABLED: "false"
DOC_VERSIONS: "latest,1.4.3,1.4.2,1.4.1,1.4.0"
APP_VERSION: "1.4.3"
CHART_VERSION: "1.4.3"
APP_VERSION: "1.4.4"
CHART_VERSION: "1.4.4"
image: debian:12-slim
......@@ -569,7 +569,6 @@ docs-registry:
- "apt-get update && apt-get install -y sed"
script:
- pip install -r ./requirements.txt
- find .docs -type f -exec sed -i -e "s/__APP_VERSION__/${APP_VERSION}/g" {} \;
- python3 .docs/docker/release.py
release-images:
......@@ -588,7 +587,7 @@ release-images:
- "ifconfig eth0 mtu 1450 up"
- "apk add make bash"
script:
- "make release"
- "make release-images"
release-chart:
stage: release
......@@ -596,13 +595,14 @@ release-chart:
only:
refs:
- /^release-.*/
except:
refs:
- release-latest
before_script:
- "echo ${CI_REGISTRY2_PASSWORD} | docker login --username ${CI_REGISTRY2_USER} --password-stdin $CI_REGISTRY2_URL"
- "echo ${CI_GPG_KEYRING} | base64 -d > ~/keyring.gpg"
- "apk add sed helm curl"
- "helm plugin install https://github.com/sigstore/helm-sigstore"
script:
- "helm package --sign --key 'Martin Weise' ./helm/dbrepo --keyring ~/keyring.gpg --destination ./build"
- "helm push ./build/dbrepo-${CHART_VERSION}.tgz oci://${CI_REGISTRY2_URL}/helm"
- "helm sigstore upload ./build/dbrepo-${CHART_VERSION}.tgz"
......@@ -617,26 +617,29 @@ release-docs:
before_script:
- "wget https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64 -O /usr/bin/yq"
- "chmod +x /usr/bin/yq"
- "apt-get update && apt-get install -y git make sed wget ssh"
- "apk add --update alpine-sdk bash sed wget openssh"
- "pip install -r ./requirements.txt"
- "mkdir -p ./final/${VERSION}/swagger"
script:
- "make gen-swagger-doc gen-lib-doc gen-docs-doc"
- "make gen-lib-doc gen-docs-doc"
- "cp -r ./lib/python/docs/build/html ./final/${VERSION}/sphinx" # sphinx
- "cp .docs/.swagger/api.yaml ./final/${VERSION}/swagger/api.yaml" # swagger
- "cp .docs/.swagger/swagger-ui.html ./final/${VERSION}/swagger/index.html" # swagger
- "cp .docs/.swagger/custom.css ./final/${VERSION}/swagger/custom.css" # swagger
- "cp -r ./site ./final/${VERSION}" # mkdocs
- "cp -r ./site/* ./final/${VERSION}" # mkdocs
- eval $(ssh-agent -s)
- "mkdir -p /root/.ssh"
- echo "$CI_KEY_PRIVATE" > /root/.ssh/id_rsa && chmod 0600 /root/.ssh/id_rsa
- echo "$CI_KEY_PUBLIC" > /root/.ssh/id_rsa.pub
- echo "$CI_DOC_ID" > ~/.ssh/known_hosts
- tar czf ./final.tar.gz ./final
- "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa final.tar.gz $CI_DOC_USER@$CI_DOC_IP:final.tar.gz"
- "ssh -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa $CI_DOC_USER@$CI_DOC_IP 'rm -rf /system/user/ifs/infrastructures/public_html/dbrepo/*; tar xzf ./final.tar.gz; rm -f ./final.tar.gz; cp -r ./final/* /system/user/ifs/infrastructures/public_html/dbrepo; rm -rf ./final'"
- "ssh -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa $CI_DOC_USER@$CI_DOC_IP 'rm -rf /system/user/ifs/infrastructures/public_html/dbrepo/*; tar xzf ./final.tar.gz; rm -f ./final.tar.gz; cp -r ./final/* /system/user/ifs/infrastructures/public_html/dbrepo/${VERSION}; rm -rf ./final'"
release-libs:
stage: release
image: docker.io/python:3.11-alpine
when: manual
only:
refs:
- /^release-.*/
......
Makefile
+ 1
− 2
View file @ bd5de40e
......@@ -2,8 +2,7 @@
APP_VERSION ?= 1.4.3
CHART_VERSION ?= 1.4.3
REPOSITORY_1_URL ?= docker.io/dbrepo
REPOSITORY_2_URL ?= s210.dl.hpc.tuwien.ac.at/dbrepo
REPOSITORY_URL ?= docker.io/dbrepo
.PHONY: all
all: help
......
build-docs.sh
+ 3
− 2
View file @ bd5de40e
......@@ -88,6 +88,7 @@ done
# finalization
echo "==================================================="
echo "Moving default version $APP_VERSION docs to /"
cp -r ./final/${APP_VERSION}/* ./final/
echo "Moving HTML redirect and JSON versions to /"
cp ./final/${APP_VERSION}/redirect.html ./final/index.html
cp ./final/${APP_VERSION}/versions.json ./final/versions.json
echo "==================================================="
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment