.docs/.prometheus/metrics-generate.sh

+#!/bin/bash
+mvn -f ./dbrepo-metadata-service/pom.xml install
+mvn -f ./dbrepo-data-service/pom.xml -pl rest-service -Dtest="PrometheusEndpointMvcTest" test
+mvn -f ./dbrepo-metadata-service/pom.xml -pl rest-service -Dtest="PrometheusEndpointMvcTest" test

.docs/.swagger/api-analyse.yaml

-components:
-  securitySchemes:
-    basicAuth:
-      in: header
-      scheme: basic
-      type: http
-    bearerAuth:
-      bearerFormat: JWT
-      in: header
-      scheme: bearer
-      type: http
-externalDocs:
-  description: Sourcecode Documentation
-  url: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/
-info:
-  contact:
-    email: andreas.rauber@tuwien.ac.at
-    name: Prof. Andreas Rauber
-  description: Service that analyses data structures
-  license:
-    name: Apache 2.0
-    url: https://www.apache.org/licenses/LICENSE-2.0
-  title: Database Repository Analyse Service API
-  version: __APPVERSION__
-openapi: 3.0.0
-paths:
-  /api/analyse/database/{database_id}/table/{table_id}/statistics:
-    get:
-      operationId: analyse_table_stat
-      parameters:
-        - example: 1
-          in: path
-          name: database_id
-          required: true
-          schema:
-            format: int64
-            type: integer
-        - example: 1
-          in: path
-          name: table_id
-          required: true
-          schema:
-            format: int64
-            type: integer
-      responses:
-        '202':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/TableStats'
-          description: Determined statistics
-        '400':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ErrorDto'
-          description: Missing parameters
-        '404':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ErrorDto'
-          description: Table not found
-      security:
-        - bearerAuth: []
-        - basicAuth: []
-      summary: Determine table statistics
-      tags:
-        - analyse-endpoint
-  /api/analyse/datatypes:
-    get:
-      consumes:
-        - application/json
-      description: This is a simple API which returns the datatypes of a (path) csv
-        file
-      operationId: analyse_datatypes
-      parameters:
-        - example: filename_s3_key
-          in: query
-          name: filename
-          required: true
-          schema:
-            type: string
-        - example: ','
-          in: query
-          name: separator
-          required: true
-          schema:
-            type: string
-        - example: 'false'
-          in: query
-          name: enum
-          required: false
-          schema:
-            type: boolean
-        - example: '2.5'
-          in: query
-          name: enum_tol
-          required: false
-          schema:
-            type: float
-      produces:
-        - application/json
-      responses:
-        '202':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/DataTypesDto'
-          description: Determined data types successfully
-        '400':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ErrorDto'
-          description: Failed to determine data types
-        '404':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ErrorDto'
-          description: Failed to find file in Storage Service
-        '500':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ErrorDto'
-          description: Unexpected system error
-      summary: Determine datatypes
-      tags:
-        - analyse-endpoint
-  /api/analyse/keys:
-    get:
-      consumes:
-        - application/json
-      description: This is a simple API which returns the primary keys + ranking of
-        a (path) csv file
-      operationId: analyse_keys
-      parameters:
-        - example: filename_s3_key
-          in: query
-          name: filename
-          required: true
-          schema:
-            type: string
-        - example: ','
-          in: query
-          name: separator
-          required: true
-          schema:
-            type: string
-      produces:
-        - application/json
-      responses:
-        '202':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/KeysDto'
-          description: Determined keys successfully
-        '400':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ErrorDto'
-          description: Failed to determine keys
-        '404':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ErrorDto'
-          description: Failed to find file in Storage Service or is empty
-        '500':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ErrorDto'
-          description: Unexpected system error
-      summary: Determine primary keys
-      tags:
-        - analyse-endpoint
-  /health:
-    get:
-      consumes:
-        - application/json
-      description: This is a simple API which checks if the application is healthy
-      produces:
-        - application/json
-      responses:
-        '200':
-          description: OK
-          schema:
-            properties:
-              status:
-                example: UP
-                type: string
-            type: object
-      summary: Check if application is running
-      tags:
-        - health-endpoint
-servers:
-  - description: Generated server url
-    url: http://localhost:5000
-  - description: Sandbox
-    url: https://test.dbrepo.tuwien.ac.at

.docs/.swagger/api-search.yaml

-components:
-  securitySchemes:
-    basicAuth:
-      in: header
-      scheme: basic
-      type: http
-    bearerAuth:
-      bearerFormat: JWT
-      in: header
-      scheme: bearer
-      type: http
-externalDocs:
-  description: Sourcecode Documentation
-  url: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/
-info:
-  contact:
-    email: andreas.rauber@tuwien.ac.at
-    name: Prof. Andreas Rauber
-  description: Service that searches the search database
-  license:
-    name: Apache 2.0
-    url: https://www.apache.org/licenses/LICENSE-2.0
-  title: Database Repository Search Service API
-  version: __APPVERSION__
-openapi: 3.0.0
-paths:
-  /api/search:
-    get:
-      consumes:
-        - application/json
-      description: Performs a fuzzy search
-      operationId: post_fuzzy_search
-      parameters:
-        - in: query
-          required: true
-          schema:
-            properties:
-              q:
-                example: air quality
-                type: string
-            type: string
-      produces:
-        - application/json
-      responses:
-        '200':
-          content:
-            application/json:
-              schema:
-                properties:
-                  results:
-                    items:
-                      type: object
-                    type: array
-                type: object
-          description: OK, contains the elements formatted as an array of JSON arrays
-        '415':
-          description: Wrong accept type
-      summary: Performs a fuzzy search
-      tags:
-        - search-endpoint
-  /api/search/database/{database_id}:
-    delete:
-      consumes:
-        - application/json
-      description: Deletes a database
-      operationId: delete_database
-      produces:
-        - application/json
-      responses:
-        '202':
-          content:
-            application/json:
-              schema:
-                properties:
-                  id:
-                    example: 1
-                    implementation: int64
-                    type: integer
-                required:
-                  - id
-                type: object
-          description: Deleted database successfully
-        '404':
-          content:
-            application/json:
-              schema:
-                properties:
-                  message:
-                    example: Message
-                    type: string
-                  success:
-                    example: false
-                    type: boolean
-                required:
-                  - success
-                  - message
-                type: object
-          description: Database not found
-      security:
-        - bearerAuth: []
-        - basicAuth: []
-      summary: Deletes a database
-      tags:
-        - database-endpoint
-    put:
-      consumes:
-        - application/json
-      description: Updates a database
-      operationId: update_database
-      parameters:
-        - in: body
-          name: body
-          required: true
-          schema:
-            properties:
-              internal_name:
-                example: air_quality_abcd
-                type: string
-              name:
-                example: Air Quality
-                type: string
-            type: object
-      produces:
-        - application/json
-      responses:
-        '202':
-          content:
-            application/json:
-              schema:
-                properties:
-                  id:
-                    example: 1
-                    implementation: int64
-                    type: integer
-                required:
-                  - id
-                type: object
-          description: Updated database successfully
-        '400':
-          content:
-            application/json:
-              schema:
-                properties:
-                  message:
-                    example: Message
-                    type: string
-                  success:
-                    example: false
-                    type: boolean
-                required:
-                  - success
-                  - message
-                type: object
-          description: Invalid schema
-        '404':
-          content:
-            application/json:
-              schema:
-                properties:
-                  message:
-                    example: Message
-                    type: string
-                  success:
-                    example: false
-                    type: boolean
-                required:
-                  - success
-                  - message
-                type: object
-          description: Database not found
-      security:
-        - bearerAuth: []
-        - basicAuth: []
-      summary: Updates a database
-      tags:
-        - database-endpoint
-  /api/search/{index}:
-    get:
-      consumes:
-        - application/json
-      description: Gets the index
-      operationId: get_index
-      parameters:
-        - description: The search type.
-          in: path
-          name: type
-          required: true
-          schema:
-            enum:
-              - database
-              - table
-              - view
-              - column
-              - user
-              - identifier
-              - concept
-              - unit
-            type: string
-        - in: body
-          name: body
-          required: true
-          schema:
-            properties:
-              field_value_pairs:
-                type: object
-              search_term:
-                example: air quality
-                type: string
-              t1:
-                example: 0
-                type: integer
-              t2:
-                example: 100
-                type: integer
-            type: object
-      produces:
-        - application/json
-      responses:
-        '200':
-          content:
-            application/json:
-              schema:
-                properties:
-                  results:
-                    items:
-                      type: object
-                    type: array
-                  type:
-                    description: Same as the requested type
-                    enum:
-                      - database
-                      - table
-                      - view
-                      - column
-                      - user
-                      - identifier
-                      - concept
-                      - unit
-                    type: string
-                type: object
-          description: OK, contains the elements formatted as an array of JSON arrays
-      summary: Gets the index
-      tags:
-        - search-endpoint
-  /api/search/{type}:
-    post:
-      consumes:
-        - application/json
-      description: Performs a general search
-      operationId: post_general_search
-      parameters:
-        - description: The search type.
-          in: path
-          name: type
-          required: true
-          schema:
-            enum:
-              - database
-              - table
-              - view
-              - column
-              - user
-              - identifier
-              - concept
-              - unit
-            type: string
-        - in: query
-          name: t1
-          schema:
-            type: integer
-        - in: query
-          name: t2
-          schema:
-            type: integer
-        - in: body
-          name: body
-          required: true
-          schema:
-            properties:
-              field_value_pairs:
-                type: object
-              search_term:
-                example: air quality
-                type: string
-            type: object
-      produces:
-        - application/json
-      responses:
-        '200':
-          content:
-            application/json:
-              schema:
-                properties:
-                  results:
-                    items:
-                      type: object
-                    type: array
-                  type:
-                    description: Same as the requested type
-                    enum:
-                      - database
-                      - table
-                      - view
-                      - column
-                      - user
-                      - identifier
-                      - concept
-                      - unit
-                    type: string
-                type: object
-          description: OK, contains the elements formatted as an array of JSON arrays
-      summary: Performs a general search
-      tags:
-        - search-endpoint
-  /api/search/{type}/fields:
-    get:
-      operationId: get_fields
-      parameters:
-        - description: The search type.
-          in: path
-          name: type
-          required: true
-          schema:
-            enum:
-              - database
-              - table
-              - view
-              - column
-              - user
-              - identifier
-              - concept
-              - unit
-            type: string
-      responses:
-        '200':
-          content:
-            application/json:
-              schema:
-                properties:
-                  results:
-                    items:
-                      properties:
-                        attr_friendly_name:
-                          example: Name
-                          type: string
-                        attr_name:
-                          example: name
-                          type: string
-                        type:
-                          description: OpenSearch data types.
-                          example: string
-                          type: string
-                      type: object
-                    type: array
-                type: object
-          description: List of fields
-        '404':
-          description: Invalid type.
-      summary: Get searchable fields
-      tags:
-        - search-endpoint
-  /health:
-    get:
-      consumes:
-        - application/json
-      description: 'Return UP if the instance is ready to serve connections.
-
-        '
-      produces:
-        - application/json
-      responses:
-        '200':
-          description: OK, service is up and running
-          schema:
-            $ref: '#/definitions/Column'
-        '404':
-          description: Service is not yet ready
-      summary: Return a healthcheck
-      tags:
-        - actuator
-servers:
-  - description: Generated server url
-    url: http://localhost:4000
-  - description: Sandbox
-    url: https://test.dbrepo.tuwien.ac.at

.docs/.swagger/api-sidecar.yaml

-components:
-  securitySchemes:
-    basicAuth:
-      in: header
-      scheme: basic
-      type: http
-    bearerAuth:
-      bearerFormat: JWT
-      in: header
-      scheme: bearer
-      type: http
-externalDocs:
-  description: Sourcecode Documentation
-  url: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/__APPVERSION__/
-info:
-  contact:
-    email: andreas.rauber@tuwien.ac.at
-    name: Prof. Andreas Rauber
-  description: Sidecar that downloads the import .csv file
-  license:
-    name: Apache 2.0
-    url: https://www.apache.org/licenses/LICENSE-2.0
-  title: Database Repository Data Database sidecar API
-  version: __APPVERSION__
-openapi: 3.0.0
-paths:
-  /health:
-    get:
-      consumes:
-        - application/json
-      description: 'Return UP if the instance is ready to serve connections.
-
-        '
-      produces:
-        - application/json
-      responses:
-        '200':
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/Health'
-          description: OK, service is up and running
-        '404':
-          description: Service is not yet ready
-      summary: Return a healthcheck
-      tags:
-        - actuator
-  /sidecar/export/{filename}:
-    post:
-      consumes:
-        - application/json
-      description: 'Exports a specific .csv file to the Storage Service via S3
-
-        '
-      parameters:
-        - description: Name of the object file to export to the Storage Service
-          in: path
-          name: filename
-          required: true
-      produces:
-        - application/json
-      responses:
-        '202':
-          content: {}
-          description: Exported the .csv
-        '400':
-          description: The Storage Service could not be contacted or .csv was not
-            found.
-      security:
-        - bearerAuth: []
-        - basicAuth: []
-      summary: Exports a .csv to the Storage Service
-      tags:
-        - sidecar
-  /sidecar/import/{filename}:
-    post:
-      consumes:
-        - application/json
-      description: 'Imports a specific .csv file from the Storage Service via S3
-
-        '
-      parameters:
-        - description: Name of the object file to import from the Storage Service
-          in: path
-          name: filename
-          required: true
-      produces:
-        - application/json
-      responses:
-        '202':
-          content: {}
-          description: Imported the .csv
-        '400':
-          description: The Storage Service could not be contacted or .csv was not
-            found.
-      security:
-        - bearerAuth: []
-        - basicAuth: []
-      summary: Imports a .csv from the Storage Service
-      tags:
-        - sidecar
-servers:
-  - description: Generated server url
-    url: http://localhost:8080
-  - description: Sandbox
-    url: https://test.dbrepo.tuwien.ac.at

.docs/.swagger/api-upload.yaml

-openapi: 3.0.0
-info:
-  title: Database Repository Upload Service API
-  description: Service that manages the uploads
-  contact:
-    name: Prof. Andreas Rauber
-    email: andreas.rauber@tuwien.ac.at
-  license:
-    name: Apache 2.0
-    url: https://www.apache.org/licenses/LICENSE-2.0
-  version: __APPVERSION__
-externalDocs:
-  description: Sourcecode Documentation
-  url: https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services
-servers:
-  - url: http://localhost:1080
-    description: Generated server url
-  - url: https://test.dbrepo.tuwien.ac.at
-    description: Sandbox
-paths:
-  /api/upload/files:
-    post:
-      tags:
-        - upload-endpoint
-      summary: Uploads a file
-      operationId: upload
-      responses:
-        "201":
-          description: "Successfully uploaded a file"
-      security: { }
\ No newline at end of file

.docs/.swagger/api.base.yaml

+components:
+  securitySchemes:
+    basicAuth:
+      in: header
+      scheme: basic
+      type: http
+    bearerAuth:
+      bearerFormat: JWT
+      in: header
+      scheme: bearer
+      type: http
+externalDocs:
+  description: Sourcecode Documentation
+  url: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/
+info:
+  contact:
+    email: andreas.rauber@tuwien.ac.at
+    name: Prof. Andreas Rauber
+  description: The REST API
+  license:
+    name: Apache 2.0
+    url: https://www.apache.org/licenses/LICENSE-2.0
+  title: DBRepo REST API
+  version: 1.4.4
+openapi: 3.1.0
+servers:
+  - description: Test Instance
+    url: https://test.dbrepo.tuwien.ac.at
+  - description: Local Instance
+    url: http://localhost

.docs/.swagger/custom.css

@@ -3,6 +3,10 @@ body {
     background: #eee;
 }
 
+#logo {
+    margin-top: 1rem;
+}
+
 .scheme-container {
     background: none !important;
     box-shadow: none !important;

.docs/.swagger/swagger-generate.sh

@@ -9,7 +9,7 @@ services[3305]=sidecar
 function retrieve () {
   if [[ "$2" == analyse ]] || [[ "$2" == search ]] || [[ "$2" == sidecar ]]; then
     echo "... retrieve json api from localhost:$1"
-    curl -sSL "http://localhost:$1/api-$2.json" | yq -y > "./.docs/.swagger/api-$2.yaml"
+    curl -sSL "http://localhost:$1/api-$2.json" | yq -p=json > "./.docs/.swagger/api-$2.yaml"
   else
     echo "... retrieve yaml api from localhost:$1"
     curl -sSL "http://localhost:$1/v3/api-docs.yaml" > "./.docs/.swagger/api-$2.yaml"

.docs/.swagger/swagger-site.sh

 #!/bin/bash
 declare -A services
-services[1080]=upload
+#services[1080]=upload
 services[4000]=search
 services[5000]=analyse
 services[9093]=data
 services[9099]=metadata
 services[3305]=sidecar
 
-# ensure target directories are present
-echo "ensure target directory ./site are present"
-mkdir -p ./site
+rm -f ./tmp.yaml
+mkdir -p ./site/swagger
+touch ./tmp.yaml
 
-# extract static site
-echo "extract static site .docs/.swagger/dist.tar.gz"
+# -> build paths: map
 for key in "${!services[@]}"; do
-  mkdir -p ./site/${services[$key]}
-  echo "extract static site ./swagger-ui.html -> ./site/${services[$key]}"
-  cp .docs/.swagger/swagger-ui.html ./site/${services[$key]}/index.html
-  cp .docs/.swagger/custom.css ./site/${services[$key]}/custom.css
-  sed -i -e "s/__SERVICENAME__/${services[$key]^} Service/g" ./site/${services[$key]}/index.html
-  cp ".docs/.swagger/api-${services[$key]}.yaml" "./site/${services[$key]}/api.yaml"
+  cat .docs/.swagger/api-${services[$key]}.yaml | yq .paths >> ./tmp.yaml
 done
+
+# -> merge with api.base.yaml into final api.yaml
+yq ".paths *= load(\"tmp.yaml\")" .docs/.swagger/api.base.yaml > .docs/.swagger/api.yaml
+

.docs/.swagger/swagger-ui.html

@@ -3,16 +3,22 @@
 <head>
     <meta charset="utf-8"/>
     <meta name="viewport" content="width=device-width, initial-scale=1"/>
-    <meta name="description" content="__SERVICENAME__ API description in OpenAPI 3.0"/>
-    <title>__SERVICENAME__ API</title>
-    <link rel="stylesheet" href="https://unpkg.com/swagger-ui-dist@5.11.1/swagger-ui.css"/>
+    <meta name="description" content="DBRepo REST API description in OpenAPI 3.0"/>
+    <title>DBRepo REST API</title>
+    <link rel="stylesheet" href="https://unpkg.com/swagger-ui-dist@5.17.12/swagger-ui.css"/>
     <link rel="stylesheet" href="./custom.css"/>
     <link rel="icon" href="https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/master/.docs/images/signet_white.png" />
 </head>
 <body>
+<div class="swagger-ui">
+    <div class="wrapper">
+        <img id="logo" width="300" alt="DBRepo Logo"
+             src="https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/master/dbrepo-ui/public/logo.png" />
+    </div>
+</div>
 <div id="swagger-ui"></div>
-<script src="https://unpkg.com/swagger-ui-dist@5.11.1/swagger-ui-bundle.js" crossorigin></script>
-<script src="https://unpkg.com/swagger-ui-dist@5.11.1/swagger-ui-standalone-preset.js" crossorigin></script>
+<script src="https://unpkg.com/swagger-ui-dist@5.17.12/swagger-ui-bundle.js" crossorigin></script>
+<script src="https://unpkg.com/swagger-ui-dist@5.17.12/swagger-ui-standalone-preset.js" crossorigin></script>
 <script>
     window.onload = () => {
       window.ui = SwaggerUIBundle({

.docs/system-services-analyse.md → .docs/api/analyse-service.md

@@ -2,8 +2,6 @@
 author: Martin Weise
 ---
 
-# Analyse Service
-
 ## tl;dr
 
 !!! debug "Debug Information"

.docs/system-services-authentication.md → .docs/api/auth-service.md

@@ -2,8 +2,6 @@
 author: Martin Weise
 ---
 
-# Authentication Service
-
 ## tl;dr
 
 !!! debug "Debug Information"
@@ -38,7 +36,7 @@ We organize the roles into default- and escalated composite roles. There are thr
 Each of the composite role has a set of other associated composite roles.
 
 <figure markdown>
-![](images/groups-roles.png)
+![Grouped Roles](../images/groups-roles.png)
 <figcaption>Three groups (Researchers, Developers, Data Stewards) and their composite roles associated.</figcaption>
 </figure>
 

.docs/system-services-broker.md → .docs/api/broker-service.md

@@ -2,8 +2,6 @@
 author: Martin Weise
 ---
 
-# Broker Service
-
 ## tl;dr
 
 !!! debug "Debug Information"
@@ -40,14 +38,14 @@ queue `dbrepo`, connected with a binding of `dbrepo.#` which routes all tuples w
 the dot!) to this queue.
 
 <figure markdown>
-   ![Data ingest](images/queue-quorum.png)
+   ![Data ingest](../images/queue-quorum.png)
    <figcaption>Replicated quorum queue dbrepo in a cluster with three nodes</figcaption>
 </figure>
 
 The consumer takes care of writing it to the correct table in the [Data Service](../system-services-data).
 
 <figure markdown>
-   ![Data ingest](images/exchange-binding.png)
+   ![Data ingest](../images/exchange-binding.png)
    <figcaption>Architecture Broker Service</figcaption>
 </figure>
 

.docs/system-databases-data.md → .docs/api/data-db.md

@@ -2,10 +2,6 @@
 author: Martin Weise
 ---
 
-# Data Database
-
-## tl;dr
-
 !!! debug "Debug Information"
 
     Image: [`bitnami/mariadb-galera:11.2.2-debian-11-r0`](https://hub.docker.com/r/bitnami/mariadb-galera)
@@ -64,7 +60,7 @@ the [`boto3`](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html
 natively.
 
 <figure markdown>
-![Sidecar architecture detailed](images/architecture-data-db.svg)
+![Sidecar architecture detailed](../images/architecture-data-db.svg)
 <figcaption>Sidecar that handles the CSV-file upload/download.</figcaption>
 </figure>
 

.docs/system-services-data.md → .docs/api/data-service.md

@@ -2,8 +2,6 @@
 author: Martin Weise
 ---
 
-# Data Service
-
 ## tl;dr
 
 !!! debug "Debug Information"
@@ -26,8 +24,6 @@ Data Service up.
 
 ## Limitations
 
-* No clear differentiation for data-handling endpoints from the [Metadata Service](../system-services-metadata) yet.
-
 !!! question "Do you miss functionality? Do these limitations affect you?"
 
     We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get

.docs/system-services-gateway.md → .docs/api/gateway-service.md

@@ -2,8 +2,6 @@
 author: Martin Weise
 ---
 
-# Gateway Service
-
 ## tl;dr
 
 !!! debug "Debug Information"