diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index a7496bc4f47838b9944b1c844a3b7d98e32cf1f3..2e2a41a065da226ac4dc9a3076d69be04d6f0d21 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -17,8 +17,8 @@ services: image: docker.io/bitnami/mariadb:11.1.3-debian-11-r6 volumes: - metadata-db-data:/bitnami/mariadb - - ./dbrepo-metadata-db/setup-schema.sql:/docker-entrypoint-initdb.d/1_setup-schema.sql - - ./dbrepo-metadata-db/setup-data.sql:/docker-entrypoint-initdb.d/2_setup-data.sql + - ./dist/setup-schema.sql:/docker-entrypoint-initdb.d/1_setup-schema.sql + - ./dist/setup-data.sql:/docker-entrypoint-initdb.d/2_setup-data.sql ports: - "3306:3306" environment: @@ -192,8 +192,6 @@ services: volumes: - ./dist/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf - ./dist/enabled_plugins:/etc/rabbitmq/enabled_plugins - - ./dist/cert.pem:/app/cert.pem - - ./dist/pubkey.pem:/app/pubkey.pem - ./dist/definitions.json:/app/definitions.json - broker-service-data:/bitnami/rabbitmq/mnesia depends_on: diff --git a/.docs/examples/air.md b/.docs/examples/air.md index 1e4d9ddeddf83d5861ad421ba8d00f7397508f46..7c290f047c1bc45712fd52da3c00ff16a2b54d9a 100644 --- a/.docs/examples/air.md +++ b/.docs/examples/air.md @@ -5,7 +5,7 @@ author: Martin Weise ## tl;dr [:fontawesome-solid-database: Dataset](https://handle.stage.datacite.org/10.82556/gd17-aq82){ .md-button .md-button--primary target="_blank" } -[:material-file-document: Archive](https://doi.org/10.48436/mtha8-w2406){ .md-button .md-button--secondary target="_blank" } +[:simple-grafana: Dashboard](https://dbrepo1.ec.tuwien.ac.at/admin/grafana/d/FLB9eAv4z/airquality){ .md-button .md-button--secondary target="_blank" } ## Description @@ -14,14 +14,25 @@ in Vienna, spanning the years from 1980 to 2021. The data was provided by the Um original form in this record. This record forms the basis of an analysis carried out in a bachelor's thesis at the TU Wien. -## Solution +<figure markdown> + +<figcaption>Figure 1: Grafana dashboard visualizing the dataset.</figcaption> +</figure> +The analysis was carried out in a Jupyter Notebook hosted by our IT-department +[JupyterHub](https://science.datalab.tuwien.ac.at/) as part of TU Wien's virtual research environment. <figure markdown> { .img-border } -<figcaption>Figure 1: Jupyter Notebook accessing data on DBRepo using the Python Library.</figcaption> +<figcaption>Figure 2: Jupyter Notebook accessing data on DBRepo using the Python Library.</figcaption> </figure> +## Solution + +One of the first use-cases of importing external data into DBRepo which was provided as .csv flat file. We developed a +database schema and a web scraper that scrapes live air quality data from the +[public map](https://luft.umweltbundesamt.at/pub/map_chart/index.pl) of the environment agency of Austria. + ## DBRepo Features - [x] Import complex dataset diff --git a/.docs/examples/survey.md b/.docs/examples/survey.md index 88eea632e066e4ce3ed250d47e7ba6e5c46b5c8f..c30f5efb67efc827474dd7a33cba99ae73af323d 100644 --- a/.docs/examples/survey.md +++ b/.docs/examples/survey.md @@ -9,22 +9,25 @@ author: Martin Weise ## Description -This digital record contains historical air pollution and air quality data from approximately 20 air monitoring stations -in Vienna, spanning the years from 1980 to 2021. The data was provided by the Umweltbundesamt and is stored in its -original form in this record. This record forms the basis of an analysis carried out in a bachelor's thesis at the TU -Wien. +As part of a literature study, the research unit of data science has collected data on 47 Trusted Research Environments +(TREs) who enable analysis of confidential data under strict security assertions who protect the data with technical, +organizational and legal measures from (accidentally) being leaked outside the facility. The literature study shows that +47 TREs worldwide provide access to confidential data of which two-thirds provide data themselves (n=32, 68%), +predominantly via secure remote access (n=46, 98%). ## Solution +We designed a database schema that allows collection of the data with correct primary key and foreign-key relationships. +Three defined views allow for a simpler exploration of the study data. The analysis of the data was performed in TU +Wien's virtual research environment using [JupyterHub](https://science.datalab.tuwien.ac.at/) as well as the chart <figure markdown> -{ .img-border } +{ .img-border } <figcaption>Figure 1: Jupyter Notebook accessing data on DBRepo using the Python Library.</figcaption> </figure> ## DBRepo Features -- [x] Import complex dataset - [x] System versioning - [x] Subset exploration - [x] Aggregated views @@ -33,6 +36,6 @@ Wien. ## Acknowledgement -This work was part of a cooperation with the [Umweltbundesamt](https://www.umweltbundesamt.at/). +This work was part of a cooperation with the [Research Unit of Data Science](https://informatics.tuwien.ac.at/orgs/e194-04). -<img src="../../images/logos/umweltbundesamt.png" width=100 /> \ No newline at end of file +<img src="../../images/logos/ds-ifs.png" width=100 /> \ No newline at end of file diff --git a/.docs/images/logos/ds-ifs.png b/.docs/images/logos/ds-ifs.png new file mode 100644 index 0000000000000000000000000000000000000000..0bce76feea8eabadf5c63422d927d01284cd5c91 Binary files /dev/null and b/.docs/images/logos/ds-ifs.png differ diff --git a/.docs/images/screenshots/air-dashboard.png b/.docs/images/screenshots/air-dashboard.png new file mode 100644 index 0000000000000000000000000000000000000000..309a1c689f2c7955d36cdd0e724bfb71027db562 Binary files /dev/null and b/.docs/images/screenshots/air-dashboard.png differ diff --git a/.docs/images/screenshots/tre-notebook.png b/.docs/images/screenshots/tre-notebook.png new file mode 100644 index 0000000000000000000000000000000000000000..26b335113aeea58e1d50200973e60946fe681a73 Binary files /dev/null and b/.docs/images/screenshots/tre-notebook.png differ diff --git a/.docs/installation.md b/.docs/installation.md index 2561b4944b65de05a6f79b55502ad2d61ad15906..e2f67570ce19ee547bf259d4674a48e05413762a 100644 --- a/.docs/installation.md +++ b/.docs/installation.md @@ -44,7 +44,48 @@ are *not* recommended and not tested. ## Custom Install -TBD +In case you prefer a customized install, start by downloading the `docker-compose.yml` file used to define the services: + +```bash +curl -O docker-compose.yml -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/.docker/docker-compose.yml +``` + +Create the folder `dist/` that hold necessary configuration files and download the Metadata Database schema and initial +data to display the created Data Database container: + +```bash +mkdir -p dist +curl -O dist/setup-schema.sql -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-metadata-db/setup-schema.sql +curl -O dist/setup-data.sql -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-metadata-db/setup-data.sql +``` + +Download the Broker Service configuration files: + +```bash +curl -O dist/rabbitmq.conf -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-broker-service/rabbitmq.conf +curl -O dist/enabled_plugins -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-broker-service/enabled_plugins +curl -O dist/definitions.json -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-broker-service/definitions.json +``` + +!!! warning "Default admin user credentials" + + Note that you need to change the default user credentials `fda:fda` of the Broker Service by setting `users.0.name` + and `users.0.password_hash` of the `definitions.json` file. The `password_hash` can be created by executing + `./helm/dbrepo/hack/generate-rabbitmq-pw.sh <your_password>`. + +Download the Gateway Service configuration file (or integrate it into your existing NGINX reverse proxy config): + +```bash +curl -O dist/dbrepo.conf -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-gateway-service/dbrepo.conf +``` + +Download the S3 configuration for the Storage Service: + +```bash +curl -O dist/s3_config.conf -sSL https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-1.4.4/dbrepo-storage-service/s3_config.conf +``` + +Continue the custom install by customizing the [User Interface](../api/ui). ## Architecture diff --git a/dbrepo-broker-service/cert.pem b/dbrepo-broker-service/cert.pem deleted file mode 100644 index e66555558cfa59a12b3d754401e9f84fef7299e4..0000000000000000000000000000000000000000 --- a/dbrepo-broker-service/cert.pem +++ /dev/null @@ -1,3 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICmzCCAYMCBgGG3GWyBTANBgkqhkiG9w0BAQsFADARMQ8wDQYDVQQDDAZkYnJlcG8wHhcNMjMwMzEzMTkxMzE3WhcNMzMwMzEzMTkxNDU3WjARMQ8wDQYDVQQDDAZkYnJlcG8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCqqcdDYFZZb28M0tEJzEP77FmD/Xqioyj9zWX6VwUSOMAgmMmn8eqs9hT9T0a+q4YTo9tUW1PNbUpwprA5b4Uk04DcIajxDVMUR/PjcHytmkqwVskq9AZW/Vngdoo+8tSbuIybwe/3Vwt266hbHpDcM97a+DXcYooRl7tQWCEX7RP27wQrMD9epDQ6IgKayZg9vC9/03dsIqwH9jXQRiZlFvwiEKhX2aY7lPGBaCK414JO00K/Z49iov9TRa/IYVbSt5qwgrx6DcqsBSPwOnI6A85UGfeUEZ/7coVJiL7RvBlsllapsL9eWTbQajVh94k9Ei3sibEPbtH+U2OAM78zAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAASnN1Cuif1sdfEK2kWAURSXGJCohCROLWdKFjaeHPRaEfpbFJsgxW0Yj3nwX5O3bUlOWoTyENwnXSsXMQsqnNi+At32CKaKO8+AkhAbgQL9F0B+KeJwmYv3cUj5N/LYkJjBvZBzUZ4Ugu5dcxH0k7AktLAIwimkyEnxTNolOA3UyrGGpREr8MCKWVr10RFuOpF/0CsJNNwbHXzalO9D756EUcRWZ9VSg6QVNso0YYRKTnILWDn9hcTRnqGy3SHo3anFTqQZ+BB57YbgFWy6udC0LYRB3zdp6zNti87eu/VEymiDY/mmo1AB8Tm0b6vxFz4AKcL3ax5qS6YnZ9efSzk= ------END CERTIFICATE----- \ No newline at end of file diff --git a/dbrepo-broker-service/pubkey.pem b/dbrepo-broker-service/pubkey.pem deleted file mode 100644 index 9e4e9308e115194765587bb0451e85f98996f1c6..0000000000000000000000000000000000000000 --- a/dbrepo-broker-service/pubkey.pem +++ /dev/null @@ -1,3 +0,0 @@ ------BEGIN RSA PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB ------END RSA PUBLIC KEY----- \ No newline at end of file diff --git a/dbrepo-broker-service/rabbitmq.conf b/dbrepo-broker-service/rabbitmq.conf index 9efa167ba41d77d9c91a12fa63382a4d626f0b90..42a58131f6b2032d24facf25206be7c7d7af858e 100644 --- a/dbrepo-broker-service/rabbitmq.conf +++ b/dbrepo-broker-service/rabbitmq.conf @@ -18,8 +18,8 @@ log.console.level = warning # Obviously your authentication server cannot vouch for itself, so you'll need another backend with at least one user in # it. You should probably use the internal database -auth_backends.1 = rabbit_auth_backend_oauth2 -auth_backends.2 = rabbit_auth_backend_internal +#auth_backends.1 = rabbit_auth_backend_oauth2 +auth_backends.1 = rabbit_auth_backend_internal # management.oauth_enabled = true # management.oauth_client_id = rabbitmq-client @@ -28,10 +28,10 @@ auth_backends.2 = rabbit_auth_backend_internal # management.oauth_provider_url = http://localhost/api/auth/realms/dbrepo # OAuth 2.0 files -auth_oauth2.resource_server_id = rabbitmq -auth_oauth2.preferred_username_claims.1 = client_id -auth_oauth2.default_key = t2OCeCheJ9uwoBbNQjG_nN6WKiLcceTIAZmiTbGODFM -auth_oauth2.signing_keys.t2OCeCheJ9uwoBbNQjG_nN6WKiLcceTIAZmiTbGODFM = /app/cert.pem -auth_oauth2.signing_keys.id2 = /app/pubkey.pem -auth_oauth2.algorithms.1 = HS256 -auth_oauth2.algorithms.2 = RS256 +#auth_oauth2.resource_server_id = rabbitmq +#auth_oauth2.preferred_username_claims.1 = client_id +#auth_oauth2.default_key = t2OCeCheJ9uwoBbNQjG_nN6WKiLcceTIAZmiTbGODFM +#auth_oauth2.signing_keys.t2OCeCheJ9uwoBbNQjG_nN6WKiLcceTIAZmiTbGODFM = /app/cert.pem +#auth_oauth2.signing_keys.id2 = /app/pubkey.pem +#auth_oauth2.algorithms.1 = HS256 +#auth_oauth2.algorithms.2 = RS256 diff --git a/docker-compose.yml b/docker-compose.yml index 65d23f7e45b36a82df4cad73cccf6424b9413e61..00d896b8e11ac2d3a315a3f328be0f463b05aec7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -207,8 +207,6 @@ services: volumes: - ./dbrepo-broker-service/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf - ./dbrepo-broker-service/enabled_plugins:/etc/rabbitmq/enabled_plugins - - ./dbrepo-broker-service/cert.pem:/app/cert.pem - - ./dbrepo-broker-service/pubkey.pem:/app/pubkey.pem - ./dbrepo-broker-service/definitions.json:/app/definitions.json - broker-service-data:/bitnami/rabbitmq/mnesia depends_on: diff --git a/helm/dbrepo/values.yaml b/helm/dbrepo/values.yaml index b45f832b3a56b3f72a533ef221dfa1a755da8ddd..818410a82bde2c85c504126130f89fb5fc33ea1c 100644 --- a/helm/dbrepo/values.yaml +++ b/helm/dbrepo/values.yaml @@ -125,8 +125,6 @@ authservice: ## @section Data Database datadb: - global: - storageClass: test ## @param datadb.enabled Enable the Data Database. enabled: true ## @skip datadb.fullnameOverride diff --git a/install.sh b/install.sh index a11a7c3b2380bca183e2a7987df405eef8212259..5d3731beb674ef3403f4feba7a60e98d5fce0fa1 100644 --- a/install.sh +++ b/install.sh @@ -63,8 +63,6 @@ curl -sSL -o ./dist/1_setup-schema.sql "https://gitlab.phaidra.org/fair-data-aus curl -sSL -o ./dist/2_setup-data.sql "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-metadata-db/setup-data.sql" curl -sSL -o ./dist/rabbitmq.conf "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/rabbitmq.conf" curl -sSL -o ./dist/enabled_plugins "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/enabled_plugins" -curl -sSL -o ./dist/cert.pem "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/cert.pem" -curl -sSL -o ./dist/pubkey.pem "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/pubkey.pem" curl -sSL -o ./dist/definitions.json "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-broker-service/definitions.json" curl -sSL -o ./dist/dbrepo.conf "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-gateway-service/dbrepo.conf" curl -sSL -o ./dist/opensearch_dashboards.yml "https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/release-${VERSION}/dbrepo-search-db/opensearch_dashboards.yml"