Merge branch 'master' into dev

.docs/docker/_header.md

@@ -10,7 +10,7 @@
 
 # Supported tags
 
-* [`1.4.3`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.3/dbrepo-DIR/Dockerfile/)
+* [`1.4.4`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-1.4.3/dbrepo-DIR/Dockerfile/)
 * [`latest`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/release-latest/dbrepo-DIR/Dockerfile/)
 
 # Non-supported tags
@@ -29,8 +29,8 @@
 
 * **Source of this description:**
 
-  [docs repo's `.docs/docker` directory](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tree/release-1.4.3/.docs/docker)
-  ([history](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/commits/release-1.4.3/.docs/docker))
+  [docs repo's `.docs/docker` directory](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/tree/release-1.4.4/.docs/docker)
+  ([history](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/commits/release-1.4.34/.docs/docker))
 
 # What is DBRepo?
 

.docs/examples/covid19.md

----
-author: Martin Weise
----
-
-## tl;dr
-
-tbd
-
-## Description
-
-TBD
-
-## Solution
-
-TBD
-
-## DBRepo Features
-
-- [x] Large Dataset ≈15GiB
-- [x] Subset Citation using PID
-- [x] External access from Grafana Dashboard
\ No newline at end of file

.docs/examples/hazard.md

@@ -9,11 +9,34 @@ author: Martin Weise
 
 ## Description
 
-TBD
+This data base contains concentrations of hazardous substances and other water quality parameters in different
+environmental compartments:
+
+* river water (water and suspended sediments)
+* ground water
+* waste water (treated and untreated) and sewage sludge
+* storm water runoff from combined and separate sewer systems
+* atmospheric deposition
+* soil
+
+Data from many different data sources were collected, checked and combined and meta data were harmonized to allow for
+a combined data evaluation.
 
 ## Solution
 
-TBD
+We imported the database from the [archive](https://doi.org/10.48436/yaecs-dgr27) repository, converted the PostgreSQL
+language code to MariaDB:
+
+* `"current_user"()` to `current_user()`
+* `SEQUENCE CACHE 1` to `NOCACHE` because of MariaDB Galera distribution not being able to cache sequences
+* `(0)::double precision` to `0.0`
+* `character varying` to `text` because MariaDB needs sizes, `text` data type can have arbitrary size
+* `!~~` to `NOT LIKE`
+* `ANY ARRAY` to `<array> OR <array2>` because of different MariaDB syntax
+
+The complex nature of the views (i.e. `SELECT-FROM-SELECT` statements) required a logical overhaul of how DBRepo obtains
+view metadata. As a consequence, we removed the SQL parser that tried to parse the metadata up to a depth of 10 to a
+simple query to the `information_schema` that is maintained by the database engine.
 
 ## DBRepo Features
 

.docs/examples/transportation.md

@@ -2,10 +2,27 @@
 author: Martin Weise
 ---
 
+## tl;dr
+
+[:fontawesome-solid-database:  Dataset](https://dbrepo1.ec.tuwien.ac.at/pid/14){ .md-button .md-button--primary target="_blank" }
+[:simple-grafana:  Dashboard](https://dbrepo1.ec.tuwien.ac.at/admin/grafana/d/8meGcJD4k/wiener-linien){ .md-button .md-button--secondary target="_blank" }
+
 ## Description
 
-TBD
+The Subway Transportation Data-Dataset is a comprehensive and dynamic collection of data that captures the intricate 
+details of the city's public transportation system. This dataset encompasses a wide array of information, including bus 
+and tram schedules, subway routes, ticketing details, and real-time updates on vehicle locations.
 
 ## Solution
 
-TBD
+We wrote an algorithm that parses open data (available) information from Wiener Linien, Vienna's public transportation
\ No newline at end of file
+agency directly and feeds it, after some cleaning, into DBRepo on a 5-minute interval.
+
+![Subway Transportation Data Dashboard](images/screenshots/transportation-dashboard.png)
+
+## DBRepo Features
+
+- [x] Dynamic data (live data)
+- [x] System versioning
+- [x] Subset exploration
+- [x] External visualization of the database
\ No newline at end of file

.docs/redirect.html

@@ -5,16 +5,16 @@
 <head>
     <meta charset="UTF-8">
     <title>Redirect Notice</title>
-    <meta http-equiv="Refresh" content="0; url='/infrastructures/dbrepo/__APPVERSION__/'" />
+    <meta http-equiv="Refresh" content="0; url='/infrastructures/dbrepo/1.4.4/'" />
 </head>
 <body>
 <h1>Redirect Notice</h1>
 <p>
-    This page should automatically open the documentation for version <code>__APPVERSION__</code>. In case this page does not load the site is
+    This page should automatically open the documentation for version <code>1.4.4</code>. In case this page does not load the site is
     available at:
 </p>
 <p>
-    <a href="/infrastructures/dbrepo/__APPVERSION__/">/infrastructures/dbrepo/__APPVERSION__/</a>
+    <a href="/infrastructures/dbrepo/1.4.4/">/infrastructures/dbrepo/1.4.4/</a>
 </p>
 </body>
 </html>
\ No newline at end of file

.gitlab-ci.yml

@@ -4,7 +4,6 @@ variables:
   TRIVY_CACHE_DIR: ".trivycache/"
   DOCKER_HOST: "unix:///var/run/dind/docker.sock"
   TESTCONTAINERS_RYUK_DISABLED: "false"
-  DOC_VERSIONS: "latest,1.4.3,1.4.2,1.4.1,1.4.0"
   APP_VERSION: "1.4.4"
   CHART_VERSION: "1.4.4"
 
@@ -601,10 +600,10 @@ release-chart:
   before_script:
     - "echo ${CI_REGISTRY2_PASSWORD} | docker login --username ${CI_REGISTRY2_USER} --password-stdin $CI_REGISTRY2_URL"
     - "apk add sed helm curl"
+    - "helm package ./helm/dbrepo --destination ./build"
     - "helm plugin install https://github.com/sigstore/helm-sigstore"
   script:
     - "helm push ./build/dbrepo-${CHART_VERSION}.tgz oci://${CI_REGISTRY2_URL}/helm"
-    - "helm sigstore upload ./build/dbrepo-${CHART_VERSION}.tgz"
 
 release-docs:
   stage: release
@@ -632,7 +631,9 @@ release-docs:
     - echo "$CI_DOC_ID" > ~/.ssh/known_hosts
     - tar czf ./final.tar.gz ./final
     - "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa final.tar.gz $CI_DOC_USER@$CI_DOC_IP:final.tar.gz"
-    - "ssh -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa $CI_DOC_USER@$CI_DOC_IP 'rm -rf /system/user/ifs/infrastructures/public_html/dbrepo/*; tar xzf ./final.tar.gz; rm -f ./final.tar.gz; cp -r ./final/* /system/user/ifs/infrastructures/public_html/dbrepo/${APP_VERSION}; rm -rf ./final'"
+    - "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa versions.json $CI_DOC_USER@$CI_DOC_IP:/system/user/ifs/infrastructures/public_html/dbrepo/versions.json"
+    - "scp -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa .docs/redirect.html $CI_DOC_USER@$CI_DOC_IP:/system/user/ifs/infrastructures/public_html/dbrepo/index.html"
+    - "ssh -oHostKeyAlgorithms=+ssh-rsa -oPubkeyAcceptedAlgorithms=+ssh-rsa $CI_DOC_USER@$CI_DOC_IP 'rm -rf /system/user/ifs/infrastructures/public_html/dbrepo/${APP_VERSION}; tar xzf ./final.tar.gz; rm -f ./final.tar.gz; cp -r ./final/* /system/user/ifs/infrastructures/public_html/dbrepo/${APP_VERSION}; rm -rf ./final'"
 
 release-libs:
   stage: release

CONTRIBUTING.md

@@ -25,3 +25,14 @@ Explore the available labels in order to proper categorize it and get the fastes
 
 If the contribution it's a bugfix, a little feature or documentation improvement that could be implemented in, lets say,
 a couple of days at maximum, one could go directly for a PR. It's fine.
+
+### Release checklist
+
+- [ ] Change variables `APP_VERSION` and `CHART_VERSION` in CI/CD file `.gitlab-ci.yml`
+- [ ] Change Helm chart variables in `helm/dbrepo/Chart.yaml` and update the chart README.md and values.schema.json for artifact hub with `make gen-helm-doc`
+- [ ] Change Python library version in `lib/python/setup.py` and `lib/python/pyproject.toml` for PyPI
+- [ ] Change the supported tags list in `.docs/docker/_header.md` for docker hub
+- [ ] Change the maven version in the metadata & data services:
+  - `mvn -f ./dbrepo-metadata-service/pom.xml versions:set -DnewVersion=VERSION`
+  - `mvn -f ./dbrepo-data-service/pom.xml versions:set -DnewVersion=VERSION`
+- [ ] Change the versions in `versions.json` for the generated website
\ No newline at end of file

Makefile

 .PHONY: all
 
-APP_VERSION ?= 1.4.3
-CHART_VERSION ?= 1.4.3
+APP_VERSION ?= 1.4.4
+CHART_VERSION ?= 1.4.4
 REPOSITORY_URL ?= docker.io/dbrepo
 
 .PHONY: all

docker-compose.yml

@@ -137,7 +137,7 @@ services:
       DELETED_RECORD: "${DELETED_RECORD:-persistent}"
       GRANULARITY: "${GRANULARITY:-YYYY-MM-DDThh:mm:ssZ}"
       JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"
-      LOG_LEVEL: trace
+      LOG_LEVEL: ${LOG_LEVEL:-info}
       METADATA_DB: "${METADATA_DB:-dbrepo}"
       METADATA_HOST: "${METADATA_HOST:-metadata-db}"
       METADATA_JDBC_EXTRA_ARGS: "${METADATA_JDBC_EXTRA_ARGS:-}"
@@ -482,7 +482,7 @@ services:
       GRANT_DEFAULT_READ: "${GRANT_DEFAULT_READ:-SELECT}"
       GRANT_DEFAULT_WRITE: "${GRANT_DEFAULT_WRITE:-SELECT, CREATE, CREATE VIEW, CREATE ROUTINE, CREATE TEMPORARY TABLES, LOCK TABLES, INDEX, TRIGGER, INSERT, UPDATE, DELETE}"
       JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"
-      LOG_LEVEL: ${LOG_LEVEL:-trace}
+      LOG_LEVEL: ${LOG_LEVEL:-info}
       MIN_CONCURRENT_CONSUMERS: ${MIN_CONCURRENT_CONSUMERS:-1}
       MAX_CONCURRENT_CONSUMERS: ${MAX_CONCURRENT_CONSUMERS:-5}
       QUEUE_NAME: ${QUEUE_NAME:-dbrepo}

mkdocs.yml

@@ -44,11 +44,7 @@ nav:
     - UI:
       - Customization: api/ui.md
   - Examples:
-    - COVID-19 Tweets: examples/covid19.md
     - Hazardous Materials: examples/hazard.md
-    - Influenza Monitoring: examples/influenza.md
-    - Manufacturing Data: examples/manufacturing.md
-    - Power Usage: examples/power.md
     - Transportation Monitoring: examples/transportation.md
   - publications.md
   - contact.md

tmp/.gitignore

-HELP.md
-target/
-out/
-!.mvn/wrapper/maven-wrapper.jar
-!**/src/main/**/target/
-!**/src/test/**/target/
-!**/src/main/**/out/
-!**/src/test/**/out/
-
-### Environment ###
-.env
-
-### Generated ###
-ready
-mapping.xml
-*.versionsBackup
-
-### STS ###
-.apt_generated
-.classpath
-.factorypath
-.project
-.settings
-.springBeans
-.sts4-cache
-
-### IntelliJ IDEA ###
-.idea
-*.iws
-*.iml
-*.ipr
-
-### NetBeans ###
-/nbproject/private/
-/nbbuild/
-/dist/
-/nbdist/
-/.nb-gradle/
-build/
-!**/src/main/**/build/
-!**/src/test/**/build/
-
-### VS Code ###
-.vscode/

tmp/Dockerfile

-###### FIRST STAGE ######
-FROM dbrepo-metadata-service:build as dependency
-MAINTAINER Martin Weise <martin.weise@tuwien.ac.at>
-
-###### SECOND STAGE ######
-FROM maven:3-openjdk-17 as build
-MAINTAINER Martin Weise <martin.weise@tuwien.ac.at>
-
-COPY ./pom.xml ./
-
-RUN mvn -fn -B dependency:go-offline
-
-COPY --from=dependency /root/.m2/repository/at/tuwien /root/.m2/repository/at/tuwien
-
-COPY ./api ./api
-COPY ./querystore ./querystore
-COPY ./report ./report
-COPY ./rest-service ./rest-service
-COPY ./services ./services
-
-# Make sure it compiles
-RUN mvn clean package -DskipTests
-
-###### THIRD STAGE ######
-FROM eclipse-temurin:17-jdk as runtime
-MAINTAINER Martin Weise <martin.weise@tuwien.ac.at>
-
-WORKDIR /app
-
-COPY --from=build ./rest-service/target/rest-service-*.jar ./data-service.jar
-
-EXPOSE 9093
-
-ENTRYPOINT ["java", "-Dlog4j2.formatMsgNoLookups=true",  "-jar", "./data-service.jar"]
\ No newline at end of file

tmp/README.md

-# Data Service
-
-## Test
-
-Run all unit and integration tests and create an HTML+TXT coverage report located in the `report` module:
-
-```bash
-mvn -pl rest-service clean test verify
-```
-
-Or run only tests 
-in [`DatabaseServiceIntegrationTest.java`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/blob/master/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java):
-
-```bash
-mvn -pl rest-service -Dtest="DatabaseServiceIntegrationTest" clean test
-```
-
-## Run
-
-Start the Metadata Database, Data Database, Broker Service before and then run the Data Service:
-
-```bash
-mvn -pl rest-service clean spring-boot:run -Dspring-boot.run.profiles=local
-```
-
-### Endpoints
-
-#### Actuator
-
-- Info: http://localhost:9093/actuator/info
-- Health: http://localhost:9093/actuator/health
-    - Readiness: http://localhost:9093/actuator/health/readiness
-    - Liveness: http://localhost:9093/actuator/health/liveness
-- Prometheus: http://localhost:9093/actuator/prometheus
-
-#### Swagger UI
-
-- Swagger UI: http://localhost:9093/swagger-ui/index.html
-
-#### OpenAPI
-
-- OpenAPI v3 as .yaml: http://localhost:9093/v3/api-docs.yaml
\ No newline at end of file

tmp/api/pom.xml

-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <parent>
-        <groupId>at.tuwien</groupId>
-        <artifactId>dbrepo-data-service</artifactId>
-        <version>1.4.3</version>
-    </parent>
-
-    <artifactId>dbrepo-data-service-api</artifactId>
-    <name>dbrepo-data-service-api</name>
-    <version>1.4.3</version>
-
-    <dependencies/>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <configuration>
-                    <source>${java.version}</source>
-                    <target>${java.version}</target>
-                    <annotationProcessorPaths>
-                        <path>
-                            <groupId>org.projectlombok</groupId>
-                            <artifactId>lombok</artifactId>
-                            <version>${lombok.version}</version>
-                        </path>
-                    </annotationProcessorPaths>
-                </configuration>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
\ No newline at end of file

tmp/api/src/main/java/at/tuwien/ExportResourceDto.java

-package at.tuwien;
-
-import lombok.*;
-import org.springframework.core.io.InputStreamResource;
-
-@Getter
-@Setter
-@ToString
-@Builder
-@AllArgsConstructor
-@NoArgsConstructor
-public class ExportResourceDto {
-
-    private InputStreamResource resource;
-
-    private String filename;
-
-}

tmp/api/src/main/java/at/tuwien/api/SortTypeDto.java

-package at.tuwien.api;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public enum SortTypeDto {
-
-    @JsonProperty("asc")
-    ASC("asc"),
-
-    @JsonProperty("desc")
-    DESC("desc");
-
-    private String type;
-
-    SortTypeDto(String type) {
-        this.type = type;
-    }
-
-    public String toString() {
-        return this.type;
-    }
-}

tmp/api/src/main/java/at/tuwien/api/amqp/ChannelDetailsDto.java

-package at.tuwien.api.amqp;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import jakarta.validation.constraints.NotNull;
-import lombok.*;
-import lombok.extern.jackson.Jacksonized;
-
-@Getter
-@Setter
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@Jacksonized
-@ToString
-public class ChannelDetailsDto {
-
-    @NotNull
-    @JsonProperty("connection_name")
-    private String connectionName;
-
-    @NotNull
-    private String name;
-
-    @NotNull
-    private String node;
-
-    @NotNull
-    @JsonProperty("number")
-    private Integer number;
-
-    @NotNull
-    @JsonProperty("peer_host")
-    private String peerHost;
-
-    @NotNull
-    @JsonProperty("peer_port")
-    private Integer peerPort;
-
-    @NotNull
-    private String user;
-
-}

tmp/api/src/main/java/at/tuwien/api/amqp/ConsumerDto.java

-package at.tuwien.api.amqp;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import lombok.*;
-
-import jakarta.validation.constraints.NotNull;
-import lombok.extern.jackson.Jacksonized;
-
-@Getter
-@Setter
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@Jacksonized
-@ToString
-public class ConsumerDto {
-
-    @NotNull
-    @JsonProperty("ack_required")
-    private Boolean ackRequired;
-
-    @NotNull
-    private Boolean active;
-
-    @NotNull
-    @JsonProperty("activity_status")
-    private String activityStatus;
-
-    @NotNull
-    @JsonProperty("channel_details")
-    private ChannelDetailsDto channelDetails;
-
-    @NotNull
-    @JsonProperty("consumer_tag")
-    private String consumerTag;
-
-    @NotNull
-    private Boolean exclusive;
-
-    @NotNull
-    @JsonProperty("prefetch_count")
-    private Integer prefetchCount;
-
-    @NotNull
-    private QueueBriefDto queue;
-
-}

tmp/api/src/main/java/at/tuwien/api/amqp/CreateExchangeDto.java

-package at.tuwien.api.amqp;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.swagger.v3.oas.annotations.Parameter;
-import lombok.*;
-
-import jakarta.validation.constraints.NotBlank;
-import jakarta.validation.constraints.NotNull;
-import lombok.extern.jackson.Jacksonized;
-
-@Getter
-@Setter
-@Builder
-@NoArgsConstructor
-@AllArgsConstructor
-@Jacksonized
-@ToString
-public class CreateExchangeDto {
-
-    @NotNull
-    @JsonProperty("auto_delete")
-    private Boolean autoDelete;
-
-    @NotNull
-    private Boolean durable;
-
-    @NotNull
-    private Boolean internal;
-
-    @NotBlank
-    private String type;
-
-}