diff --git a/.docs/examples/covid19.md b/.docs/examples/covid19.md deleted file mode 100644 index e5db773195075167bb353d473777fa56916ca52a..0000000000000000000000000000000000000000 --- a/.docs/examples/covid19.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -author: Martin Weise ---- - -## tl;dr - -tbd - -## Description - -TBD - -## Solution - -TBD - -## DBRepo Features - -- [x] Large Dataset ≈15GiB -- [x] Subset Citation using PID -- [x] External access from Grafana Dashboard \ No newline at end of file diff --git a/.docs/examples/hazard.md b/.docs/examples/hazard.md index 9f434d8eae0f42b1703658980ac0e393036a0a2a..56c56d99c5c9fcdc9b594fe673b2293b8d276d73 100644 --- a/.docs/examples/hazard.md +++ b/.docs/examples/hazard.md @@ -9,11 +9,34 @@ author: Martin Weise ## Description -TBD +This data base contains concentrations of hazardous substances and other water quality parameters in different +environmental compartments: + +* river water (water and suspended sediments) +* ground water +* waste water (treated and untreated) and sewage sludge +* storm water runoff from combined and separate sewer systems +* atmospheric deposition +* soil + +Data from many different data sources were collected, checked and combined and meta data were harmonized to allow for +a combined data evaluation. ## Solution -TBD +We imported the database from the [archive](https://doi.org/10.48436/yaecs-dgr27) repository, converted the PostgreSQL +language code to MariaDB: + +* `"current_user"()` to `current_user()` +* `SEQUENCE CACHE 1` to `NOCACHE` because of MariaDB Galera distribution not being able to cache sequences +* `(0)::double precision` to `0.0` +* `character varying` to `text` because MariaDB needs sizes, `text` data type can have arbitrary size +* `!~~` to `NOT LIKE` +* `ANY ARRAY` to `<array> OR <array2>` because of different MariaDB syntax + +The complex nature of the views (i.e. `SELECT-FROM-SELECT` statements) required a logical overhaul of how DBRepo obtains +view metadata. As a consequence, we removed the SQL parser that tried to parse the metadata up to a depth of 10 to a +simple query to the `information_schema` that is maintained by the database engine. ## DBRepo Features diff --git a/.docs/examples/transportation.md b/.docs/examples/transportation.md index 0aedca2e37ba1bd4b45c6a86398e06d0c568fe17..aeb8ab94ae57898cd261102e8f0adb750375ef89 100644 --- a/.docs/examples/transportation.md +++ b/.docs/examples/transportation.md @@ -2,10 +2,27 @@ author: Martin Weise --- +## tl;dr + +[:fontawesome-solid-database: Dataset](https://dbrepo1.ec.tuwien.ac.at/pid/14){ .md-button .md-button--primary target="_blank" } +[:simple-grafana: Dashboard](https://dbrepo1.ec.tuwien.ac.at/admin/grafana/d/8meGcJD4k/wiener-linien){ .md-button .md-button--secondary target="_blank" } + ## Description -TBD +The Subway Transportation Data-Dataset is a comprehensive and dynamic collection of data that captures the intricate +details of the city's public transportation system. This dataset encompasses a wide array of information, including bus +and tram schedules, subway routes, ticketing details, and real-time updates on vehicle locations. ## Solution -TBD \ No newline at end of file +We wrote an algorithm that parses open data (available) information from Wiener Linien, Vienna's public transportation +agency directly and feeds it, after some cleaning, into DBRepo on a 5-minute interval. + + + +## DBRepo Features + +- [x] Dynamic data (live data) +- [x] System versioning +- [x] Subset exploration +- [x] External visualization of the database \ No newline at end of file diff --git a/.docs/images/screenshots/transportation-dashboard.png b/.docs/images/screenshots/transportation-dashboard.png new file mode 100644 index 0000000000000000000000000000000000000000..cf10755f4e66c44f8636f8a5db7089f2301c42ca Binary files /dev/null and b/.docs/images/screenshots/transportation-dashboard.png differ diff --git a/docker-compose.yml b/docker-compose.yml index 415c754090cad7fea2799123465f5421a04cc827..17389cad12cdd606c9c1e91d4c5d4b8fdfabf168 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -137,7 +137,7 @@ services: DELETED_RECORD: "${DELETED_RECORD:-persistent}" GRANULARITY: "${GRANULARITY:-YYYY-MM-DDThh:mm:ssZ}" JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}" - LOG_LEVEL: trace + LOG_LEVEL: ${LOG_LEVEL:-info} METADATA_DB: "${METADATA_DB:-dbrepo}" METADATA_HOST: "${METADATA_HOST:-metadata-db}" METADATA_JDBC_EXTRA_ARGS: "${METADATA_JDBC_EXTRA_ARGS:-}" @@ -482,7 +482,7 @@ services: GRANT_DEFAULT_READ: "${GRANT_DEFAULT_READ:-SELECT}" GRANT_DEFAULT_WRITE: "${GRANT_DEFAULT_WRITE:-SELECT, CREATE, CREATE VIEW, CREATE ROUTINE, CREATE TEMPORARY TABLES, LOCK TABLES, INDEX, TRIGGER, INSERT, UPDATE, DELETE}" JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}" - LOG_LEVEL: ${LOG_LEVEL:-trace} + LOG_LEVEL: ${LOG_LEVEL:-info} MIN_CONCURRENT_CONSUMERS: ${MIN_CONCURRENT_CONSUMERS:-1} MAX_CONCURRENT_CONSUMERS: ${MAX_CONCURRENT_CONSUMERS:-5} QUEUE_NAME: ${QUEUE_NAME:-dbrepo} diff --git a/mkdocs.yml b/mkdocs.yml index 1455b24f4e3fd6bfb20e5f0f855ce7cdb1d9e82f..86b83c4eeb73fad7393f6255583527d9159f7078 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -44,11 +44,7 @@ nav: - UI: - Customization: api/ui.md - Examples: - - COVID-19 Tweets: examples/covid19.md - Hazardous Materials: examples/hazard.md - - Influenza Monitoring: examples/influenza.md - - Manufacturing Data: examples/manufacturing.md - - Power Usage: examples/power.md - Transportation Monitoring: examples/transportation.md - publications.md - contact.md