diff --git a/.docs/system-services-metadata.md b/.docs/system-services-metadata.md index 5c0e1d52001eac9f4c1833214a140aa7a953d639..b7fe150c8b90b09826f66feb8a158abf21793f2a 100644 --- a/.docs/system-services-metadata.md +++ b/.docs/system-services-metadata.md @@ -71,6 +71,10 @@ execution to the raw data. Any stale queries (query that have been executed by u periodically being deleted from the query store based on the `DELETE_STALE_QUERIES_RATE` environment variable (defaults to 60 seconds). +Executing SQL queries through the Query Endpoint must fulfill some restrictions: + +* The SQL query does not contain at semicolon `;` + ### Semantics The service provides metadata to the table columns in the [Metadata Database](../system-databases-metadata) from diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index beb1f3ceab219179c2239df2511a6416a15890d4..edce253766e65fa113ab011223c93288bb5a3ba8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -71,6 +71,18 @@ build-docker: - "docker build -t dbrepo-data-service:build --target build dbrepo-data-service" - "docker compose build --parallel" +build-helm: + image: docker.io/docker:24-dind + stage: build + before_script: + - echo "$CI_REGISTRY_PASSWORD" | docker login --username "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY_URL + script: + - apk add sed helm curl + - 'sed -i -e "s/^version:.*/version: \"${CHART_VERSION}\"/g" ./helm-charts/dbrepo/Chart.yaml' + - 'sed -i -e "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/g" ./helm-charts/dbrepo/Chart.yaml' + - find ./helm-charts -type f -exec sed -i -e "s/__CHARTVERSION__/${CHART_VERSION}/g" {} \; + - helm package ./helm-charts/dbrepo --destination ./build + test-metadata-service: image: maven:3-openjdk-17 stage: test @@ -119,12 +131,13 @@ test-analyse-service: script: - "pip install pipenv" - "pipenv install gunicorn && pipenv install --dev --system --deploy" - - "cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_determine_stats.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt" + - cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_s3_client.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" artifacts: when: always paths: - ./dbrepo-analyse-service/report.xml + - ./dbrepo-analyse-service/coverage.txt expire_in: 1 days reports: junit: ./dbrepo-analyse-service/report.xml diff --git a/dbrepo-analyse-service/pytest.ini b/dbrepo-analyse-service/pytest.ini index 0aa3d82bb84628e89c95d0c4611688d344d2b148..21cb1c450f4c85de4ba416e0fe68ec9a93345aee 100644 --- a/dbrepo-analyse-service/pytest.ini +++ b/dbrepo-analyse-service/pytest.ini @@ -1,4 +1,3 @@ [pytest] log_cli = 1 -log_level = info -log_disable = main \ No newline at end of file +log_level = info \ No newline at end of file diff --git a/dbrepo-metadata-db/setup-schema.sql b/dbrepo-metadata-db/setup-schema.sql index ce9488685b8592ea9169c810b075087574fc2325..b267724b05bb77f8079e7bf65f2dcf1995266455 100644 --- a/dbrepo-metadata-db/setup-schema.sql +++ b/dbrepo-metadata-db/setup-schema.sql @@ -1,550 +1,553 @@ - BEGIN; - - CREATE TABLE IF NOT EXISTS `mdb_users` - ( - id character varying(36) NOT NULL, - username character varying(255) NOT NULL, - firstname character varying(255), - lastname character varying(255), - email character varying(255) NOT NULL, - orcid character varying(255), - affiliation character varying(255), - mariadb_password character varying(255) NOT NULL, - theme_dark boolean, - PRIMARY KEY (id), - UNIQUE (username), - UNIQUE (email) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_images` - ( - id bigint NOT NULL AUTO_INCREMENT, - name character varying(255) NOT NULL, - version character varying(255) NOT NULL, - default_port integer NOT NULL, - dialect character varying(255) NOT NULL, - driver_class character varying(255) NOT NULL, - jdbc_method character varying(255) NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), - last_modified timestamp, - PRIMARY KEY (id), - UNIQUE (name, version) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_images_date` - ( - id bigint NOT NULL AUTO_INCREMENT, - iid bigint NOT NULL, - database_format character varying(255) NOT NULL, - unix_format character varying(255) NOT NULL, - example character varying(255) NOT NULL, - has_time boolean NOT NULL, - created_at timestamp NOT NULL DEFAULT NOW(), - PRIMARY KEY (id), - FOREIGN KEY (iid) REFERENCES mdb_images (id), - UNIQUE (database_format, unix_format, example) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_containers` - ( - id bigint NOT NULL AUTO_INCREMENT, - internal_name character varying(255) NOT NULL, - name character varying(255) NOT NULL, - host character varying(255) NOT NULL, - port integer NOT NULL default 3306, - ui_host character varying(255) NOT NULL default host, - ui_port integer NOT NULL default port, - ui_additional_flags text, - sidecar_host character varying(255) NOT NULL, - sidecar_port integer NOT NULL default 3305, - image_id bigint NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), - last_modified timestamp, - privileged_username character varying(255) NOT NULL, - privileged_password character varying(255) NOT NULL, - PRIMARY KEY (id), - FOREIGN KEY (image_id) REFERENCES mdb_images (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_data` - ( - ID bigint NOT NULL AUTO_INCREMENT, - PROVENANCE text, - FileEncoding text, - FileType character varying(100), - Version text, - Seperator text, - PRIMARY KEY (ID) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_licenses` - ( - identifier character varying(255) NOT NULL, - uri text NOT NULL, - PRIMARY KEY (identifier), - UNIQUE (uri(200)) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_databases` - ( - id bigint NOT NULL AUTO_INCREMENT, - cid bigint NOT NULL, - name character varying(255) NOT NULL, - internal_name character varying(255) NOT NULL, - exchange_name character varying(255) NOT NULL, - description text, - engine character varying(20), - is_public boolean NOT NULL DEFAULT TRUE, - image longblob, - created_by character varying(36), - owned_by character varying(36), - contact_person character varying(36), - created timestamp NOT NULL DEFAULT NOW(), - last_modified timestamp, - PRIMARY KEY (id), - FOREIGN KEY (cid) REFERENCES mdb_containers (id) /* currently we only support one-to-one */, - FOREIGN KEY (created_by) REFERENCES mdb_users (id), - FOREIGN KEY (owned_by) REFERENCES mdb_users (id), - FOREIGN KEY (contact_person) REFERENCES mdb_users (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_databases_subjects` - ( - dbid BIGINT NOT NULL, - subjects character varying(255) NOT NULL, - PRIMARY KEY (dbid, subjects) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_tables` - ( - ID bigint NOT NULL AUTO_INCREMENT, - tDBID bigint NOT NULL, - internal_name character varying(255) NOT NULL, - queue_name character varying(255) NOT NULL, - routing_key character varying(255) NOT NULL, - tName VARCHAR(50), - tDescription TEXT, - num_rows BIGINT, - data_length BIGINT, - max_data_length BIGINT, - avg_row_length BIGINT, - `separator` CHAR(1), - quote CHAR(1), - element_null VARCHAR(50), - skip_lines BIGINT, - element_true VARCHAR(50), - element_false VARCHAR(50), - Version TEXT, - created timestamp NOT NULL DEFAULT NOW(), - versioned boolean not null default true, - created_by character varying(36) NOT NULL, - owned_by character varying(36) NOT NULL, - last_modified timestamp, - PRIMARY KEY (ID), - FOREIGN KEY (tDBID) REFERENCES mdb_databases (id), - FOREIGN KEY (created_by) REFERENCES mdb_users (id), - FOREIGN KEY (owned_by) REFERENCES mdb_users (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_columns` - ( - ID BIGINT NOT NULL AUTO_INCREMENT, - tID BIGINT NOT NULL, - dfID BIGINT, - cName VARCHAR(100), - internal_name VARCHAR(100) NOT NULL, - alias VARCHAR(100), - Datatype ENUM ('CHAR','VARCHAR','BINARY','VARBINARY','TINYBLOB','TINYTEXT','TEXT','BLOB','MEDIUMTEXT','MEDIUMBLOB','LONGTEXT','LONGBLOB','ENUM','SET','BIT','TINYINT','BOOL','SMALLINT','MEDIUMINT','INT','BIGINT','FLOAT','DOUBLE','DECIMAL','DATE','DATETIME','TIMESTAMP','TIME','YEAR'), - length BIGINT NULL, - ordinal_position INTEGER NOT NULL, - is_primary_key BOOLEAN NOT NULL, - index_length BIGINT NULL, - size BIGINT, - d BIGINT, - auto_generated BOOLEAN DEFAULT false, - is_null_allowed BOOLEAN NOT NULL DEFAULT true, - val_min NUMERIC NULL, - val_max NUMERIC NULL, - mean NUMERIC NULL, - median NUMERIC NULL, - std_dev Numeric NULL, - created timestamp NOT NULL DEFAULT NOW(), - last_modified timestamp, - FOREIGN KEY (tID) REFERENCES mdb_tables (ID) ON DELETE CASCADE, - PRIMARY KEY (ID) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_columns_enums` - ( - id bigint NOT NULL AUTO_INCREMENT, - column_id bigint NOT NULL, - value CHARACTER VARYING(255) NOT NULL, - FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE, - PRIMARY KEY (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_columns_sets` - ( - id bigint NOT NULL AUTO_INCREMENT, - column_id bigint NOT NULL, - value CHARACTER VARYING(255) NOT NULL, - FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE, - PRIMARY KEY (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_columns_nom` - ( - tID bigint, - cID bigint, - maxlength INTEGER, - last_modified timestamp, - created timestamp NOT NULL DEFAULT NOW(), - FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID), - PRIMARY KEY (tID, cID) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_columns_cat` - ( - tID bigint, - cID bigint, - num_cat INTEGER, - -- cat_array TEXT[], - last_modified timestamp, - created timestamp NOT NULL DEFAULT NOW(), - FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID), - PRIMARY KEY (tID, cID) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key` - ( - fkid BIGINT NOT NULL AUTO_INCREMENT, - tid BIGINT NOT NULL, - rtid BIGINT NOT NULL, - on_update VARCHAR(50) NULL, - on_delete VARCHAR(50) NULL, - position INT NULL, - PRIMARY KEY (fkid), - FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE, - FOREIGN KEY (rtid) REFERENCES mdb_tables (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key_reference` - ( - id BIGINT NOT NULL AUTO_INCREMENT, - fkid BIGINT NOT NULL, - cid BIGINT NOT NULL, - rcid BIGINT NOT NULL, - PRIMARY KEY (id), - FOREIGN KEY (fkid) REFERENCES mdb_constraints_foreign_key (fkid) ON UPDATE CASCADE, - FOREIGN KEY (cid) REFERENCES mdb_columns (id), - FOREIGN KEY (rcid) REFERENCES mdb_columns (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_constraints_unique` - ( - uid BIGINT NOT NULL AUTO_INCREMENT, - tid BIGINT NOT NULL, - position INT NULL, - PRIMARY KEY (uid), - FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE - ); - - CREATE TABLE IF NOT EXISTS `mdb_constraints_unique_columns` - ( - id BIGINT NOT NULL AUTO_INCREMENT, - uid BIGINT NOT NULL, - cid BIGINT NOT NULL, - PRIMARY KEY (id), - FOREIGN KEY (uid) REFERENCES mdb_constraints_unique (uid), - FOREIGN KEY (cid) REFERENCES mdb_columns (id) ON DELETE CASCADE - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_constraints_checks` - ( - id BIGINT NOT NULL AUTO_INCREMENT, - tid BIGINT NOT NULL, - checks VARCHAR(255) NOT NULL, - PRIMARY KEY (id), - FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_concepts` - ( - id bigint NOT NULL AUTO_INCREMENT, - uri text not null, - name VARCHAR(255) null, - description TEXT null, - created timestamp NOT NULL DEFAULT NOW(), - PRIMARY KEY (id), - UNIQUE (uri(200)) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_units` - ( - id bigint NOT NULL AUTO_INCREMENT, - uri text not null, - name VARCHAR(255) null, - description TEXT null, - created timestamp NOT NULL DEFAULT NOW(), - PRIMARY KEY (id), - UNIQUE (uri(200)) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_columns_concepts` - ( - id bigint NOT NULL, - cID bigint NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), - PRIMARY KEY (id, cid), - FOREIGN KEY (cID) REFERENCES mdb_columns (ID) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_columns_units` - ( - id bigint NOT NULL, - cID bigint NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), - PRIMARY KEY (id, cID), - FOREIGN KEY (cID) REFERENCES mdb_columns (ID) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_view` - ( - id bigint NOT NULL AUTO_INCREMENT, - vdbid bigint NOT NULL, - vName VARCHAR(255) NOT NULL, - internal_name VARCHAR(255) NOT NULL, - Query TEXT NOT NULL, - query_hash VARCHAR(255) NOT NULL, - Public BOOLEAN NOT NULL, - InitialView BOOLEAN NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), - last_modified timestamp, - created_by character varying(36) NOT NULL, - PRIMARY KEY (id), - FOREIGN KEY (vdbid) REFERENCES mdb_databases (id), - FOREIGN KEY (created_by) REFERENCES mdb_users (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_banner_messages` - ( - id bigint NOT NULL AUTO_INCREMENT, - type ENUM ('ERROR', 'WARNING', 'INFO') NOT NULL default 'INFO', - message TEXT NOT NULL, - link TEXT NULL, - link_text VARCHAR(255) NULL, - display_start timestamp NULL, - display_end timestamp NULL, - PRIMARY KEY (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_ontologies` - ( - id bigint NOT NULL AUTO_INCREMENT, - prefix VARCHAR(8) NOT NULL, - uri TEXT NOT NULL, - uri_pattern TEXT, - sparql_endpoint TEXT NULL, - rdf_path TEXT NULL, - last_modified timestamp, - created timestamp NOT NULL DEFAULT NOW(), - UNIQUE (prefix), - UNIQUE (uri(200)), - PRIMARY KEY (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_view_columns` - ( - id BIGINT NOT NULL AUTO_INCREMENT, - cid BIGINT NOT NULL, - vid BIGINT NOT NULL, - position INTEGER NULL, - PRIMARY KEY (id), - FOREIGN KEY (vid) REFERENCES mdb_view (id), - FOREIGN KEY (cid) REFERENCES mdb_columns (ID) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_identifiers` - ( - id BIGINT NOT NULL AUTO_INCREMENT, - dbid BIGINT, - qid BIGINT, - vid BIGINT, - tid BIGINT, - publisher VARCHAR(255) NOT NULL, - language VARCHAR(2), - publication_year INTEGER NOT NULL, - publication_month INTEGER, - publication_day INTEGER, - identifier_type ENUM ('DATABASE', 'SUBSET', 'VIEW', 'TABLE') NOT NULL, - query TEXT, - query_normalized TEXT, - query_hash VARCHAR(255), - execution TIMESTAMP, - result_hash VARCHAR(255), - result_number BIGINT, - doi VARCHAR(255), - created TIMESTAMP NOT NULL DEFAULT NOW(), - created_by VARCHAR(36) NOT NULL, - last_modified TIMESTAMP, - PRIMARY KEY (id), /* must be a single id from persistent identifier concept */ - FOREIGN KEY (dbid) REFERENCES mdb_databases (id), - FOREIGN KEY (created_by) REFERENCES mdb_users (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_identifier_licenses` - ( - pid bigint NOT NULL, - license_id VARCHAR(255) NOT NULL, - PRIMARY KEY (pid, license_id), - FOREIGN KEY (pid) REFERENCES mdb_identifiers (id), - FOREIGN KEY (license_id) REFERENCES mdb_licenses (identifier) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_identifier_titles` - ( - id bigint NOT NULL AUTO_INCREMENT, - pid bigint NOT NULL, - title text NOT NULL, - title_type ENUM ('ALTERNATIVE_TITLE', 'SUBTITLE', 'TRANSLATED_TITLE', 'OTHER'), - language VARCHAR(2), - PRIMARY KEY (id), - FOREIGN KEY (pid) REFERENCES mdb_identifiers (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_identifier_funders` - ( - id bigint NOT NULL AUTO_INCREMENT, - pid bigint NOT NULL, - funder_name VARCHAR(255) NOT NULL, - funder_identifier TEXT, - funder_identifier_type ENUM ('CROSSREF_FUNDER_ID', 'GRID', 'ISNI', 'ROR', 'OTHER'), - scheme_uri text, - award_number VARCHAR(255), - award_title text, - language VARCHAR(255), - PRIMARY KEY (id), - FOREIGN KEY (pid) REFERENCES mdb_identifiers (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_identifier_descriptions` - ( - id bigint NOT NULL AUTO_INCREMENT, - pid bigint NOT NULL, - description text NOT NULL, - description_type ENUM ('ABSTRACT', 'METHODS', 'SERIES_INFORMATION', 'TABLE_OF_CONTENTS', 'TECHNICAL_INFO', 'OTHER'), - language VARCHAR(2), - PRIMARY KEY (id), - FOREIGN KEY (pid) REFERENCES mdb_identifiers (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_related_identifiers` - ( - id bigint NOT NULL AUTO_INCREMENT, - pid bigint NOT NULL, - value varchar(255) NOT NULL, - type varchar(255), - relation varchar(255), - PRIMARY KEY (id), /* must be a single id from persistent identifier concept */ - FOREIGN KEY (pid) REFERENCES mdb_identifiers (id), - UNIQUE (pid, value) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_identifier_creators` - ( - id bigint NOT NULL AUTO_INCREMENT, - pid bigint NOT NULL, - given_names text, - family_name text, - creator_name VARCHAR(255) NOT NULL, - name_type ENUM ('PERSONAL', 'ORGANIZATIONAL') default 'PERSONAL', - name_identifier text, - name_identifier_scheme ENUM ('ROR', 'GRID', 'ISNI', 'ORCID'), - name_identifier_scheme_uri text, - affiliation VARCHAR(255), - affiliation_identifier text, - affiliation_identifier_scheme ENUM ('ROR', 'GRID', 'ISNI'), - affiliation_identifier_scheme_uri text, - PRIMARY KEY (id), - FOREIGN KEY (pid) REFERENCES mdb_identifiers (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_feed` - ( - fDBID bigint, - fID bigint, - fUserId character varying(36) not null, - fDataID bigint REFERENCES mdb_data (ID), - created timestamp NOT NULL DEFAULT NOW(), - PRIMARY KEY (fDBID, fID, fUserId, fDataID), - FOREIGN KEY (fDBID, fID) REFERENCES mdb_tables (tDBID, ID), - FOREIGN KEY (fUserId) REFERENCES mdb_users (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_update` - ( - uUserID character varying(255) NOT NULL, - uDBID bigint NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), - PRIMARY KEY (uUserID, uDBID), - FOREIGN KEY (uDBID) REFERENCES mdb_databases (id) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_access` - ( - aUserID character varying(255) NOT NULL, - aDBID bigint REFERENCES mdb_databases (id), - attime TIMESTAMP, - download BOOLEAN, - created timestamp NOT NULL DEFAULT NOW(), - PRIMARY KEY (aUserID, aDBID) - ) WITH SYSTEM VERSIONING; - - CREATE TABLE IF NOT EXISTS `mdb_have_access` - ( - user_id character varying(36) NOT NULL, - database_id bigint REFERENCES mdb_databases (id), - access_type ENUM ('READ', 'WRITE_OWN', 'WRITE_ALL') NOT NULL, - created timestamp NOT NULL DEFAULT NOW(), - PRIMARY KEY (user_id, database_id), - FOREIGN KEY (user_id) REFERENCES mdb_users (id) - ) WITH SYSTEM VERSIONING; - - COMMIT; - BEGIN; - - INSERT INTO `mdb_licenses` (identifier, uri) - VALUES ('MIT', 'https://opensource.org/licenses/MIT'), - ('GPL-3.0-only', 'https://www.gnu.org/licenses/gpl-3.0-standalone.html'), - ('BSD-3-Clause', 'https://opensource.org/licenses/BSD-3-Clause'), - ('BSD-4-Clause', 'http://directory.fsf.org/wiki/License:BSD_4Clause'), - ('Apache-2.0', 'https://opensource.org/licenses/Apache-2.0'), - ('CC0-1.0', 'https://creativecommons.org/publicdomain/zero/1.0/legalcode'), - ('CC-BY-4.0', 'https://creativecommons.org/licenses/by/4.0/legalcode'); - - INSERT INTO `mdb_images` (name, version, default_port, dialect, driver_class, jdbc_method) - VALUES ('mariadb', '11.1.3', 3306, 'org.hibernate.dialect.MariaDBDialect', 'org.mariadb.jdbc.Driver', 'mariadb'); - - INSERT INTO `mdb_images_date` (iid, database_format, unix_format, example, has_time) - VALUES (1, '%Y-%c-%d %H:%i:%S.%f', 'yyyy-MM-dd HH:mm:ss.SSSSSS', '2022-01-30 13:44:25.499', true), - (1, '%Y-%c-%d %H:%i:%S', 'yyyy-MM-dd HH:mm:ss', '2022-01-30 13:44:25', true), - (1, '%Y-%c-%d', 'yyyy-MM-dd', '2022-01-30', false), - (1, '%H:%i:%S', 'HH:mm:ss', '13:44:25', true); - - INSERT INTO `mdb_ontologies` (prefix, uri, uri_pattern, sparql_endpoint, rdf_path) - VALUES ('om', 'http://www.ontology-of-units-of-measure.org/resource/om-2/', - 'http://www.ontology-of-units-of-measure.org/resource/om-2/.*', null, 'rdf/om-2.0.rdf'), - ('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql', - null), - ('mo', 'http://purl.org/ontology/mo/', 'http://purl.org/ontology/mo/.*', null, null), - ('dc', 'http://purl.org/dc/elements/1.1/', null, null, null), - ('xsd', 'http://www.w3.org/2001/XMLSchema#', null, null, null), - ('tl', 'http://purl.org/NET/c4dm/timeline.owl#', null, null, null), - ('foaf', 'http://xmlns.com/foaf/0.1/', null, null, null), - ('schema', 'http://schema.org/', null, null, null), - ('rdf', 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', null, null, null), - ('rdfs', 'http://www.w3.org/2000/01/rdf-schema#', null, null, null), - ('owl', 'http://www.w3.org/2002/07/owl#', null, null, null), - ('prov', 'http://www.w3.org/ns/prov#', null, null, null), - ('db', 'http://dbpedia.org', 'http://dbpedia.org/ontology/.*', 'http://dbpedia.org/sparql', null); - COMMIT; +BEGIN; + +CREATE TABLE IF NOT EXISTS `mdb_users` +( + id character varying(36) NOT NULL, + username character varying(255) NOT NULL, + firstname character varying(255), + lastname character varying(255), + email character varying(255) NOT NULL, + orcid character varying(255), + affiliation character varying(255), + mariadb_password character varying(255) NOT NULL, + theme_dark boolean, + PRIMARY KEY (id), + UNIQUE (username), + UNIQUE (email) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_images` +( + id bigint NOT NULL AUTO_INCREMENT, + name character varying(255) NOT NULL, + version character varying(255) NOT NULL, + default_port integer NOT NULL, + dialect character varying(255) NOT NULL, + driver_class character varying(255) NOT NULL, + jdbc_method character varying(255) NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), + last_modified timestamp, + PRIMARY KEY (id), + UNIQUE (name, version) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_images_date` +( + id bigint NOT NULL AUTO_INCREMENT, + iid bigint NOT NULL, + database_format character varying(255) NOT NULL, + unix_format character varying(255) NOT NULL, + example character varying(255) NOT NULL, + has_time boolean NOT NULL, + created_at timestamp NOT NULL DEFAULT NOW(), + PRIMARY KEY (id), + FOREIGN KEY (iid) REFERENCES mdb_images (id), + UNIQUE (database_format, unix_format, example) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_containers` +( + id bigint NOT NULL AUTO_INCREMENT, + internal_name character varying(255) NOT NULL, + name character varying(255) NOT NULL, + host character varying(255) NOT NULL, + port integer NOT NULL default 3306, + ui_host character varying(255) NOT NULL default host, + ui_port integer NOT NULL default port, + ui_additional_flags text, + sidecar_host character varying(255) NOT NULL, + sidecar_port integer NOT NULL default 3305, + image_id bigint NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), + last_modified timestamp, + privileged_username character varying(255) NOT NULL, + privileged_password character varying(255) NOT NULL, + PRIMARY KEY (id), + FOREIGN KEY (image_id) REFERENCES mdb_images (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_data` +( + ID bigint NOT NULL AUTO_INCREMENT, + PROVENANCE text, + FileEncoding text, + FileType character varying(100), + Version text, + Seperator text, + PRIMARY KEY (ID) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_licenses` +( + identifier character varying(255) NOT NULL, + uri text NOT NULL, + PRIMARY KEY (identifier), + UNIQUE (uri(200)) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_databases` +( + id bigint NOT NULL AUTO_INCREMENT, + cid bigint NOT NULL, + name character varying(255) NOT NULL, + internal_name character varying(255) NOT NULL, + exchange_name character varying(255) NOT NULL, + description text, + engine character varying(20), + is_public boolean NOT NULL DEFAULT TRUE, + image longblob, + created_by character varying(36), + owned_by character varying(36), + contact_person character varying(36), + created timestamp NOT NULL DEFAULT NOW(), + last_modified timestamp, + PRIMARY KEY (id), + FOREIGN KEY (cid) REFERENCES mdb_containers (id) /* currently we only support one-to-one */, + FOREIGN KEY (created_by) REFERENCES mdb_users (id), + FOREIGN KEY (owned_by) REFERENCES mdb_users (id), + FOREIGN KEY (contact_person) REFERENCES mdb_users (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_databases_subjects` +( + dbid BIGINT NOT NULL, + subjects character varying(255) NOT NULL, + PRIMARY KEY (dbid, subjects) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_tables` +( + ID bigint NOT NULL AUTO_INCREMENT, + tDBID bigint NOT NULL, + internal_name character varying(255) NOT NULL, + queue_name character varying(255) NOT NULL, + routing_key character varying(255) NOT NULL, + tName VARCHAR(50), + tDescription TEXT, + num_rows BIGINT, + data_length BIGINT, + max_data_length BIGINT, + avg_row_length BIGINT, + `separator` CHAR(1), + quote CHAR(1), + element_null VARCHAR(50), + skip_lines BIGINT, + element_true VARCHAR(50), + element_false VARCHAR(50), + Version TEXT, + created timestamp NOT NULL DEFAULT NOW(), + versioned boolean not null default true, + created_by character varying(36) NOT NULL, + owned_by character varying(36) NOT NULL, + processed_constraints BOOLEAN NOT NULL DEFAULT false, + last_modified timestamp, + PRIMARY KEY (ID), + FOREIGN KEY (tDBID) REFERENCES mdb_databases (id), + FOREIGN KEY (created_by) REFERENCES mdb_users (id), + FOREIGN KEY (owned_by) REFERENCES mdb_users (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_columns` +( + ID BIGINT NOT NULL AUTO_INCREMENT, + tID BIGINT NOT NULL, + dfID BIGINT, + cName VARCHAR(100), + internal_name VARCHAR(100) NOT NULL, + Datatype ENUM ('CHAR','VARCHAR','BINARY','VARBINARY','TINYBLOB','TINYTEXT','TEXT','BLOB','MEDIUMTEXT','MEDIUMBLOB','LONGTEXT','LONGBLOB','ENUM','SET','BIT','TINYINT','BOOL','SMALLINT','MEDIUMINT','INT','BIGINT','FLOAT','DOUBLE','DECIMAL','DATE','DATETIME','TIMESTAMP','TIME','YEAR'), + length BIGINT NULL, + ordinal_position INTEGER NOT NULL, + is_primary_key BOOLEAN NOT NULL, + index_length BIGINT NULL, + size BIGINT, + d BIGINT, + auto_generated BOOLEAN DEFAULT false, + is_null_allowed BOOLEAN NOT NULL DEFAULT true, + val_min NUMERIC NULL, + val_max NUMERIC NULL, + mean NUMERIC NULL, + median NUMERIC NULL, + std_dev Numeric NULL, + created timestamp NOT NULL DEFAULT NOW(), + last_modified timestamp, + FOREIGN KEY (tID) REFERENCES mdb_tables (ID) ON DELETE CASCADE, + PRIMARY KEY (ID) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_columns_enums` +( + id bigint NOT NULL AUTO_INCREMENT, + column_id bigint NOT NULL, + value CHARACTER VARYING(255) NOT NULL, + FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE, + PRIMARY KEY (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_columns_sets` +( + id bigint NOT NULL AUTO_INCREMENT, + column_id bigint NOT NULL, + value CHARACTER VARYING(255) NOT NULL, + FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE, + PRIMARY KEY (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_columns_nom` +( + tID bigint, + cID bigint, + maxlength INTEGER, + last_modified timestamp, + created timestamp NOT NULL DEFAULT NOW(), + FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID), + PRIMARY KEY (tID, cID) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_columns_cat` +( + tID bigint, + cID bigint, + num_cat INTEGER, + -- cat_array TEXT[], + last_modified timestamp, + created timestamp NOT NULL DEFAULT NOW(), + FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID), + PRIMARY KEY (tID, cID) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key` +( + fkid BIGINT NOT NULL AUTO_INCREMENT, + tid BIGINT NOT NULL, + rtid BIGINT NOT NULL, + name VARCHAR(255) NOT NULL, + on_update VARCHAR(50) NULL, + on_delete VARCHAR(50) NULL, + position INT NULL, + PRIMARY KEY (fkid), + FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE, + FOREIGN KEY (rtid) REFERENCES mdb_tables (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key_reference` +( + id BIGINT NOT NULL AUTO_INCREMENT, + fkid BIGINT NOT NULL, + cid BIGINT NOT NULL, + rcid BIGINT NOT NULL, + PRIMARY KEY (id), + FOREIGN KEY (fkid) REFERENCES mdb_constraints_foreign_key (fkid) ON UPDATE CASCADE, + FOREIGN KEY (cid) REFERENCES mdb_columns (id), + FOREIGN KEY (rcid) REFERENCES mdb_columns (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_constraints_unique` +( + uid BIGINT NOT NULL AUTO_INCREMENT, + name VARCHAR(255) NOT NULL, + tid BIGINT NOT NULL, + position INT NULL, + PRIMARY KEY (uid), + FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS `mdb_constraints_unique_columns` +( + id BIGINT NOT NULL AUTO_INCREMENT, + uid BIGINT NOT NULL, + cid BIGINT NOT NULL, + PRIMARY KEY (id), + FOREIGN KEY (uid) REFERENCES mdb_constraints_unique (uid), + FOREIGN KEY (cid) REFERENCES mdb_columns (id) ON DELETE CASCADE +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_constraints_checks` +( + id BIGINT NOT NULL AUTO_INCREMENT, + tid BIGINT NOT NULL, + checks VARCHAR(255) NOT NULL, + PRIMARY KEY (id), + FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_concepts` +( + id bigint NOT NULL AUTO_INCREMENT, + uri text not null, + name VARCHAR(255) null, + description TEXT null, + created timestamp NOT NULL DEFAULT NOW(), + PRIMARY KEY (id), + UNIQUE (uri(200)) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_units` +( + id bigint NOT NULL AUTO_INCREMENT, + uri text not null, + name VARCHAR(255) null, + description TEXT null, + created timestamp NOT NULL DEFAULT NOW(), + PRIMARY KEY (id), + UNIQUE (uri(200)) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_columns_concepts` +( + id bigint NOT NULL, + cID bigint NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), + PRIMARY KEY (id, cid), + FOREIGN KEY (cID) REFERENCES mdb_columns (ID) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_columns_units` +( + id bigint NOT NULL, + cID bigint NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), + PRIMARY KEY (id, cID), + FOREIGN KEY (cID) REFERENCES mdb_columns (ID) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_view` +( + id bigint NOT NULL AUTO_INCREMENT, + vdbid bigint NOT NULL, + vName VARCHAR(255) NOT NULL, + internal_name VARCHAR(255) NOT NULL, + Query TEXT NOT NULL, + query_hash VARCHAR(255) NOT NULL, + Public BOOLEAN NOT NULL, + InitialView BOOLEAN NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), + last_modified timestamp, + created_by character varying(36) NOT NULL, + PRIMARY KEY (id), + FOREIGN KEY (vdbid) REFERENCES mdb_databases (id), + FOREIGN KEY (created_by) REFERENCES mdb_users (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_banner_messages` +( + id bigint NOT NULL AUTO_INCREMENT, + type ENUM ('ERROR', 'WARNING', 'INFO') NOT NULL default 'INFO', + message TEXT NOT NULL, + link TEXT NULL, + link_text VARCHAR(255) NULL, + display_start timestamp NULL, + display_end timestamp NULL, + PRIMARY KEY (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_ontologies` +( + id bigint NOT NULL AUTO_INCREMENT, + prefix VARCHAR(8) NOT NULL, + uri TEXT NOT NULL, + uri_pattern TEXT, + sparql_endpoint TEXT NULL, + rdf_path TEXT NULL, + last_modified timestamp, + created timestamp NOT NULL DEFAULT NOW(), + UNIQUE (prefix), + UNIQUE (uri(200)), + PRIMARY KEY (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_view_columns` +( + id BIGINT NOT NULL AUTO_INCREMENT, + cid BIGINT NOT NULL, + vid BIGINT NOT NULL, + alias VARCHAR(100), + ordinal_position INTEGER, + PRIMARY KEY (id), + FOREIGN KEY (vid) REFERENCES mdb_view (id), + FOREIGN KEY (cid) REFERENCES mdb_columns (ID) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_identifiers` +( + id BIGINT NOT NULL AUTO_INCREMENT, + dbid BIGINT, + qid BIGINT, + vid BIGINT, + tid BIGINT, + publisher VARCHAR(255) NOT NULL, + language VARCHAR(2), + publication_year INTEGER NOT NULL, + publication_month INTEGER, + publication_day INTEGER, + identifier_type ENUM ('DATABASE', 'SUBSET', 'VIEW', 'TABLE') NOT NULL, + query TEXT, + query_normalized TEXT, + query_hash VARCHAR(255), + execution TIMESTAMP, + result_hash VARCHAR(255), + result_number BIGINT, + doi VARCHAR(255), + created TIMESTAMP NOT NULL DEFAULT NOW(), + created_by VARCHAR(36) NOT NULL, + last_modified TIMESTAMP, + PRIMARY KEY (id), /* must be a single id from persistent identifier concept */ + FOREIGN KEY (dbid) REFERENCES mdb_databases (id), + FOREIGN KEY (created_by) REFERENCES mdb_users (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_identifier_licenses` +( + pid bigint NOT NULL, + license_id VARCHAR(255) NOT NULL, + PRIMARY KEY (pid, license_id), + FOREIGN KEY (pid) REFERENCES mdb_identifiers (id), + FOREIGN KEY (license_id) REFERENCES mdb_licenses (identifier) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_identifier_titles` +( + id bigint NOT NULL AUTO_INCREMENT, + pid bigint NOT NULL, + title text NOT NULL, + title_type ENUM ('ALTERNATIVE_TITLE', 'SUBTITLE', 'TRANSLATED_TITLE', 'OTHER'), + language VARCHAR(2), + PRIMARY KEY (id), + FOREIGN KEY (pid) REFERENCES mdb_identifiers (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_identifier_funders` +( + id bigint NOT NULL AUTO_INCREMENT, + pid bigint NOT NULL, + funder_name VARCHAR(255) NOT NULL, + funder_identifier TEXT, + funder_identifier_type ENUM ('CROSSREF_FUNDER_ID', 'GRID', 'ISNI', 'ROR', 'OTHER'), + scheme_uri text, + award_number VARCHAR(255), + award_title text, + language VARCHAR(255), + PRIMARY KEY (id), + FOREIGN KEY (pid) REFERENCES mdb_identifiers (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_identifier_descriptions` +( + id bigint NOT NULL AUTO_INCREMENT, + pid bigint NOT NULL, + description text NOT NULL, + description_type ENUM ('ABSTRACT', 'METHODS', 'SERIES_INFORMATION', 'TABLE_OF_CONTENTS', 'TECHNICAL_INFO', 'OTHER'), + language VARCHAR(2), + PRIMARY KEY (id), + FOREIGN KEY (pid) REFERENCES mdb_identifiers (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_related_identifiers` +( + id bigint NOT NULL AUTO_INCREMENT, + pid bigint NOT NULL, + value varchar(255) NOT NULL, + type varchar(255), + relation varchar(255), + PRIMARY KEY (id), /* must be a single id from persistent identifier concept */ + FOREIGN KEY (pid) REFERENCES mdb_identifiers (id), + UNIQUE (pid, value) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_identifier_creators` +( + id bigint NOT NULL AUTO_INCREMENT, + pid bigint NOT NULL, + given_names text, + family_name text, + creator_name VARCHAR(255) NOT NULL, + name_type ENUM ('PERSONAL', 'ORGANIZATIONAL') default 'PERSONAL', + name_identifier text, + name_identifier_scheme ENUM ('ROR', 'GRID', 'ISNI', 'ORCID'), + name_identifier_scheme_uri text, + affiliation VARCHAR(255), + affiliation_identifier text, + affiliation_identifier_scheme ENUM ('ROR', 'GRID', 'ISNI'), + affiliation_identifier_scheme_uri text, + PRIMARY KEY (id), + FOREIGN KEY (pid) REFERENCES mdb_identifiers (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_feed` +( + fDBID bigint, + fID bigint, + fUserId character varying(36) not null, + fDataID bigint REFERENCES mdb_data (ID), + created timestamp NOT NULL DEFAULT NOW(), + PRIMARY KEY (fDBID, fID, fUserId, fDataID), + FOREIGN KEY (fDBID, fID) REFERENCES mdb_tables (tDBID, ID), + FOREIGN KEY (fUserId) REFERENCES mdb_users (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_update` +( + uUserID character varying(255) NOT NULL, + uDBID bigint NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), + PRIMARY KEY (uUserID, uDBID), + FOREIGN KEY (uDBID) REFERENCES mdb_databases (id) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_access` +( + aUserID character varying(255) NOT NULL, + aDBID bigint REFERENCES mdb_databases (id), + attime TIMESTAMP, + download BOOLEAN, + created timestamp NOT NULL DEFAULT NOW(), + PRIMARY KEY (aUserID, aDBID) +) WITH SYSTEM VERSIONING; + +CREATE TABLE IF NOT EXISTS `mdb_have_access` +( + user_id character varying(36) NOT NULL, + database_id bigint REFERENCES mdb_databases (id), + access_type ENUM ('READ', 'WRITE_OWN', 'WRITE_ALL') NOT NULL, + created timestamp NOT NULL DEFAULT NOW(), + PRIMARY KEY (user_id, database_id), + FOREIGN KEY (user_id) REFERENCES mdb_users (id) +) WITH SYSTEM VERSIONING; + +COMMIT; +BEGIN; + +INSERT INTO `mdb_licenses` (identifier, uri) +VALUES ('MIT', 'https://opensource.org/licenses/MIT'), + ('GPL-3.0-only', 'https://www.gnu.org/licenses/gpl-3.0-standalone.html'), + ('BSD-3-Clause', 'https://opensource.org/licenses/BSD-3-Clause'), + ('BSD-4-Clause', 'http://directory.fsf.org/wiki/License:BSD_4Clause'), + ('Apache-2.0', 'https://opensource.org/licenses/Apache-2.0'), + ('CC0-1.0', 'https://creativecommons.org/publicdomain/zero/1.0/legalcode'), + ('CC-BY-4.0', 'https://creativecommons.org/licenses/by/4.0/legalcode'); + +INSERT INTO `mdb_images` (name, version, default_port, dialect, driver_class, jdbc_method) +VALUES ('mariadb', '11.1.3', 3306, 'org.hibernate.dialect.MariaDBDialect', 'org.mariadb.jdbc.Driver', 'mariadb'); + +INSERT INTO `mdb_images_date` (iid, database_format, unix_format, example, has_time) +VALUES (1, '%Y-%c-%d %H:%i:%S.%f', 'yyyy-MM-dd HH:mm:ss.SSSSSS', '2022-01-30 13:44:25.499', true), + (1, '%Y-%c-%d %H:%i:%S', 'yyyy-MM-dd HH:mm:ss', '2022-01-30 13:44:25', true), + (1, '%Y-%c-%d', 'yyyy-MM-dd', '2022-01-30', false), + (1, '%H:%i:%S', 'HH:mm:ss', '13:44:25', true); + +INSERT INTO `mdb_ontologies` (prefix, uri, uri_pattern, sparql_endpoint, rdf_path) +VALUES ('om', 'http://www.ontology-of-units-of-measure.org/resource/om-2/', + 'http://www.ontology-of-units-of-measure.org/resource/om-2/.*', null, 'rdf/om-2.0.rdf'), + ('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql', + null), + ('mo', 'http://purl.org/ontology/mo/', 'http://purl.org/ontology/mo/.*', null, null), + ('dc', 'http://purl.org/dc/elements/1.1/', null, null, null), + ('xsd', 'http://www.w3.org/2001/XMLSchema#', null, null, null), + ('tl', 'http://purl.org/NET/c4dm/timeline.owl#', null, null, null), + ('foaf', 'http://xmlns.com/foaf/0.1/', null, null, null), + ('schema', 'http://schema.org/', null, null, null), + ('rdf', 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', null, null, null), + ('rdfs', 'http://www.w3.org/2000/01/rdf-schema#', null, null, null), + ('owl', 'http://www.w3.org/2002/07/owl#', null, null, null), + ('prov', 'http://www.w3.org/ns/prov#', null, null, null), + ('db', 'http://dbpedia.org', 'http://dbpedia.org/ontology/.*', 'http://dbpedia.org/sparql', null); +COMMIT; diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java index d94ed58767610d7300c593a1150b75aed9b175f1..b865f7892cf74fcf1ae4a0feeec3075c87fa5817 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java @@ -43,4 +43,8 @@ public class ImportDto { @Schema(example = "\"") private Character quote; + + @JsonProperty("line_termination") + @Schema(example = "\\r\\n") + private String lineTermination; } diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/constraints/foreignKey/ForeignKeyDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/constraints/foreignKey/ForeignKeyDto.java index bcc3c6d8d139aacb3e831a2e03a3025aff7bdd88..e577b033c46dd0c1d9cc030a018d1a6f8b572ac4 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/constraints/foreignKey/ForeignKeyDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/constraints/foreignKey/ForeignKeyDto.java @@ -19,6 +19,8 @@ import java.util.List; @ToString public class ForeignKeyDto { + private String name; + @org.springframework.data.annotation.Transient private List<ColumnDto> columns; diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/View.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/View.java index 7888bb6fdde986f1e5c73213a811ac6c940fc9f6..46dbac13ca02f81f0df6df08146f5ec3c34a53b4 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/View.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/View.java @@ -111,20 +111,13 @@ public class View { return this.internalName.equals(table.getName().replace("`", "")); } - /** - * Cascade cannot be CascadeType.PERSIST since columns already exist - */ @ToString.Exclude - @ManyToMany(fetch = FetchType.LAZY, cascade = CascadeType.MERGE) - @JoinTable(name = "mdb_view_columns", - inverseJoinColumns = { - @JoinColumn(name = "cid", referencedColumnName = "id"), - }, - joinColumns = { - @JoinColumn(name = "vid", referencedColumnName = "id"), - }) - @OrderColumn(name = "position") - private List<TableColumn> columns; + @OneToMany(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE, CascadeType.PERSIST}) + @JoinColumns({ + @JoinColumn(name = "vid", referencedColumnName = "id", updatable = false) + }) + @OrderColumn(name = "ordinalPosition") + private List<ViewColumn> columns; @CreatedDate @Column(nullable = false, updatable = false, columnDefinition = "TIMESTAMP default NOW()") diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java new file mode 100644 index 0000000000000000000000000000000000000000..a74bde27fe3afd1f4ae2d1878541efdf4759773a --- /dev/null +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java @@ -0,0 +1,53 @@ +package at.tuwien.entities.database; + +import at.tuwien.entities.database.table.columns.TableColumn; +import lombok.*; +import org.hibernate.annotations.GenericGenerator; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; + +import jakarta.persistence.*; + +@Data +@Entity +@Builder(toBuilder = true) +@ToString +@AllArgsConstructor +@NoArgsConstructor +@EntityListeners(AuditingEntityListener.class) +@jakarta.persistence.Table(name = "mdb_view_columns", uniqueConstraints = { + @UniqueConstraint(columnNames = {"cid", "vid"}) +}) +public class ViewColumn implements Comparable<ViewColumn> { + + @Id + @EqualsAndHashCode.Include + @GeneratedValue(generator = "view-columns-sequence") + @GenericGenerator(name = "view-columns-sequence", strategy = "increment") + @Column(updatable = false, nullable = false) + private Long id; + + @Column(updatable = false) + private String alias; + + @Column(nullable = false) + private Integer ordinalPosition; + + @ToString.Exclude + @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE}) + @JoinColumns({ + @JoinColumn(name = "vid", referencedColumnName = "id", updatable = false) + }) + private View view; + + @ToString.Exclude + @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE}) + @JoinColumns({ + @JoinColumn(name = "cid", referencedColumnName = "id", updatable = false) + }) + private TableColumn column; + + @Override + public int compareTo(ViewColumn tableColumn) { + return Integer.compare(this.ordinalPosition, tableColumn.getOrdinalPosition()); + } +} diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/Table.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/Table.java index 5ada0ba03bf50c5e844b9b69cc2a459b4dba0403..02ab0e59410b4b28c14c7dd0a86810aae4846ecf 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/Table.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/Table.java @@ -132,6 +132,9 @@ public class Table { @Column(columnDefinition = "TIMESTAMP") private Instant lastModified; + @Column(name = "processed_constraints", nullable = false) + private Boolean processedConstraints; + @Override public boolean equals(Object o) { if (o == this) { diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java index aa7b2dcd6c1032e297969fd0408825158f06d20f..2c37b13a2de2f1134bf1733b22ffedc29d7d4ddc 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java @@ -73,7 +73,7 @@ public class TableColumn implements Comparable<TableColumn> { @Column private Long indexLength; - @Column + @Transient private String alias; @Column(name = "datatype", nullable = false, columnDefinition = "ENUM('CHAR','VARCHAR','BINARY','VARBINARY','TINYBLOB','TINYTEXT','TEXT','BLOB','MEDIUMTEXT','MEDIUMBLOB','LONGTEXT','LONGBLOB','ENUM','SET','BIT','TINYINT','BOOL','SMALLINT','MEDIUMINT','INT','BIGINT','FLOAT','DOUBLE','DECIMAL','DATE','DATETIME','TIMESTAMP','TIME','YEAR')") diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/foreignKey/ForeignKey.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/foreignKey/ForeignKey.java index 2ec4f2ba5d7913cdca0a14b298773571976bfec7..92050024e5c0d24dde314a915c4746dc315c99ef 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/foreignKey/ForeignKey.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/foreignKey/ForeignKey.java @@ -26,6 +26,9 @@ public class ForeignKey { @Column(updatable = false, nullable = false) private Long fkid; + @Column(updatable = false, nullable = false) + private String name; + @ToString.Exclude @org.springframework.data.annotation.Transient @ManyToOne(fetch = FetchType.LAZY, cascade = CascadeType.MERGE) diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/unique/Unique.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/unique/Unique.java index 8cdebed830bdc4bb940002db92d49a0a4262ead4..d373339f2d0ac4e3a70344a914882087de907119 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/unique/Unique.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/unique/Unique.java @@ -27,6 +27,9 @@ public class Unique { @Column(updatable = false, nullable = false) private Long uid; + @Column(updatable = false, nullable = false) + private String name; + @ToString.Exclude @org.springframework.data.annotation.Transient @ManyToOne(fetch = FetchType.LAZY, cascade = CascadeType.MERGE) diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/DatabaseMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/DatabaseMapper.java index 7bd1c5d0eb55eb4aa815d9662732d9547f38aef4..764d79cc29130f235f5896dff35de5967c3b8c61 100644 --- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/DatabaseMapper.java +++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/DatabaseMapper.java @@ -117,7 +117,7 @@ public interface DatabaseMapper { default PreparedStatement databaseToDatabaseMetadata(Connection connection, Database database) throws QueryMalformedException { final StringBuilder statement = new StringBuilder("SELECT t.`TABLE_NAME`, t.`TABLE_TYPE`, t.`TABLE_ROWS`, t.`AVG_ROW_LENGTH`, t.`DATA_LENGTH`, t.`MAX_DATA_LENGTH`, COALESCE(t.`CREATE_TIME`, NOW()) as `CREATE_TIME`, t.`UPDATE_TIME`, v.`VIEW_DEFINITION` FROM information_schema.TABLES t LEFT JOIN information_schema.VIEWS v ON t.`TABLE_NAME` = v.`TABLE_NAME` WHERE t.`TABLE_SCHEMA` = '") .append(database.getInternalName()) - .append("' AND t.`TABLE_TYPE` IN ('BASE TABLE', 'SYSTEM VERSIONED', 'VIEW') AND t.`TABLE_NAME` NOT IN ('qs_queries', '_tmp') AND t.`TABLE_NAME` NOT LIKE 'hs_%' AND t.`TABLE_NAME` NOT LIKE '%_temporary'"); + .append("' AND t.`TABLE_TYPE` IN ('BASE TABLE', 'SYSTEM VERSIONED', 'VIEW') AND t.`TABLE_NAME` != 'qs_queries' AND t.`TABLE_NAME` NOT LIKE 'hs_%'"); log.trace("statement={}", statement); try { return connection.prepareStatement(statement.toString()); diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java index e45ce93aaae7cd8bb8b69a53af76edc8247a3e23..536810dfd735d5b9c4f135c65119bab8779ffc21 100644 --- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java +++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java @@ -9,9 +9,12 @@ import at.tuwien.api.database.table.TableCsvDto; import at.tuwien.api.database.table.TableHistoryDto; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.View; +import at.tuwien.entities.database.ViewColumn; import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.entities.database.table.columns.TableColumnType; +import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKey; +import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKeyReference; import at.tuwien.exception.ImageNotSupportedException; import at.tuwien.exception.QueryMalformedException; import at.tuwien.exception.QueryStoreException; @@ -47,6 +50,7 @@ import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; @Mapper(componentModel = "spring", imports = {LinkedList.class}) public interface QueryMapper { @@ -122,44 +126,13 @@ public interface QueryMapper { .build(); } - default void importCsvQuery(Connection connection, Table table, ImportDto csv) throws SQLException { - final Statement statement = connection.createStatement(); - final StringBuilder query0 = new StringBuilder("CREATE TABLE `") - .append(table.getDatabase().getInternalName()) - .append("`.`") - .append(table.getInternalName()) - .append("_temporary`") - .append(" LIKE `") - .append(table.getDatabase().getInternalName()) - .append("`.`") - .append(table.getInternalName()) - .append("`;"); - log.trace("mapped create temporary table statement: {}", query0); - statement.execute(query0.toString()); - final String query1 = pathToRawInsertQuery(table, csv); - log.trace("mapped import csv statement: {}", query1); - statement.execute(query1.toString()); - final String query2 = generateInsertFromTemporaryTableSQL(table); - log.trace("mapped import table statement: {}", query2); - statement.execute(query2.toString()); - final StringBuilder query3 = new StringBuilder("DROP TABLE IF EXISTS `") - .append(table.getDatabase().getInternalName()) - .append("`.`") - .append(table.getInternalName()) - .append("_temporary`;"); - log.trace("mapped drop temporary table statement: {}", query3); - statement.execute(query3.toString()); - - } - - default String pathToRawInsertQuery(Table table, ImportDto data) { + default PreparedStatement pathToRawInsertQuery(Connection connection, Table table, ImportDto data) throws QueryMalformedException { final StringBuilder statement = new StringBuilder("LOAD DATA INFILE '/tmp/") .append(data.getLocation()) - .append("' INTO TABLE `") + .append("' REPLACE INTO TABLE `") .append(table.getDatabase().getInternalName()) .append("`.`") .append(table.getInternalName()) - .append("_temporary") .append("` CHARACTER SET utf8 FIELDS TERMINATED BY '") .append(data.getSeparator()) .append("'"); @@ -168,7 +141,10 @@ public interface QueryMapper { .append(data.getQuote()) .append("'"); } - statement.append(data.getSkipLines() != null ? (" IGNORE " + data.getSkipLines() + " LINES") : "") + statement.append(" LINES TERMINATED BY '") + .append(data.getLineTermination()) + .append("'") + .append(data.getSkipLines() != null ? (" IGNORE " + data.getSkipLines() + " LINES") : "") .append(" ("); final StringBuilder set = new StringBuilder(); int[] idx = new int[]{0}; @@ -200,7 +176,14 @@ public interface QueryMapper { statement.append(")") .append(set.length() != 0 ? (" SET " + set) : "") .append(";"); - return statement.toString(); + try { + final PreparedStatement pstmt = connection.prepareStatement(statement.toString()); + log.trace("mapped import csv query {} to prepared statement {}", table.getName(), pstmt); + return pstmt; + } catch (SQLException e) { + log.error("Failed to prepare statement {}: {}", statement, e.getMessage()); + throw new QueryMalformedException("Failed to prepare statement:" + e.getMessage(), e); + } } default void columnToBoolSet(ImportDto data, TableColumn column, StringBuilder set) { @@ -209,18 +192,14 @@ public interface QueryMapper { .append("`") .append(column.getInternalName()) .append("` = "); - if (data.getNullElement() != null) { - log.trace("import has null element present"); - set.append("IF(!STRCMP(@") - .append(column.getInternalName()) - .append(",'") - .append(data.getNullElement()) - .append("'),NULL,"); - columnToBoolSet2(data, column, set); - set.append(")"); - return; - } + log.trace("import has null element present"); + set.append("IF(!STRCMP(@") + .append(column.getInternalName()) + .append(",'") + .append(data.getNullElement()) + .append("'),NULL,"); columnToBoolSet2(data, column, set); + set.append(")"); } default void columnToBoolSet2(ImportDto data, TableColumn column, StringBuilder set) { @@ -285,19 +264,14 @@ public interface QueryMapper { .append("`") .append(column.getInternalName()) .append("` = "); - if (data.getNullElement() != null) { - log.trace("import has null element present"); - set.append("IF(STRCMP(@") - .append(column.getInternalName()) - .append(",'") - .append(data.getNullElement()) - .append("'), @") - .append(column.getInternalName()) - .append(", NULL)"); - return; - } - set.append("@") - .append(column.getInternalName()); + log.trace("import has null element present"); + set.append("IF(STRCMP(@") + .append(column.getInternalName()) + .append(",'") + .append(data.getNullElement()) + .append("'), @") + .append(column.getInternalName()) + .append(", NULL)"); } default void columnToDateSet(ImportDto data, TableColumn column, StringBuilder set) { @@ -306,24 +280,14 @@ public interface QueryMapper { .append("`") .append(column.getInternalName()) .append("` = STR_TO_DATE("); - if (data.getNullElement() != null) { - log.trace("import has null element present"); - set.append("IF(STRCMP(@") - .append(column.getInternalName()) - .append(",'") - .append(data.getNullElement()) - .append("'), @") - .append(column.getInternalName()) - .append(", NULL), '") - .append(column.getDateFormat() - .getDatabaseFormat() - .replace('\'', '\\')) - .append("')"); - return; - } - set.append("@") + log.trace("import has null element present"); + set.append("IF(STRCMP(@") + .append(column.getInternalName()) + .append(",'") + .append(data.getNullElement()) + .append("'), @") .append(column.getInternalName()) - .append(", '") + .append(", NULL), '") .append(column.getDateFormat() .getDatabaseFormat() .replace('\'', '\\')) @@ -500,18 +464,18 @@ public interface QueryMapper { for (Map.Entry<String, Object> entry : data.getKeys().entrySet()) { final Optional<TableColumn> optional = table.getColumns() .stream() - .filter(c -> c.getInternalName().equals(entry.getKey())) + .filter(c -> c.getInternalName().equals(entry.getKey().replace("`", ""))) .findFirst(); if (optional.isEmpty()) { - log.error("Failed to find column with name {}, available names: {}", entry.getKey(), data.getKeys().keySet()); - throw new QueryMalformedException("Failed to find column"); + log.error("Failed to find column with name {} in table {}", entry.getKey(), table.getInternalName()); + throw new QueryMalformedException("Failed to find column with name " + entry.getKey() + " in table " + table.getInternalName()); } prepareStatementWithColumnTypeObject(pstmt, optional.get().getColumnType(), i++, entry.getValue()); } return pstmt; } catch (SQLException e) { - log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage()); - throw new QueryMalformedException("Failed to prepare statement", e); + log.error("Failed to prepare statement {}: {}", statement, e.getMessage()); + throw new QueryMalformedException("Failed to prepare statement: " + e.getMessage(), e); } } @@ -619,12 +583,6 @@ public interface QueryMapper { return columnsToRawFindAllQuery(table.getInternalName(), table.getColumns(), timestamp, size, page); } - default String viewToRawFindAllQuery(View view, Long size, Long page) { - log.trace("mapping view to find all query, view={}, size={}, page={}", view, size, page); - /* param check */ - return columnsToRawFindAllQuery(view.getInternalName(), view.getColumns(), null, size, page); - } - private String columnsToRawFindAllQuery(String tableName, List<TableColumn> columns, Instant timestamp, Long size, Long page) { final int[] idx = new int[]{0}; final StringBuilder statement = new StringBuilder("SELECT "); @@ -640,15 +598,13 @@ public interface QueryMapper { .append(LocalDateTime.ofInstant(timestamp, ZoneId.of("UTC"))) .append("'"); } - if (size != null && page != null) { - log.trace("pagination size/limit of {}", size); - statement.append(" LIMIT ") - .append(size); - log.trace("pagination page/offset of {}", page); - statement.append(" OFFSET ") - .append(page * size) - .append(";"); - } + log.trace("pagination size/limit of {}", size); + statement.append(" LIMIT ") + .append(size); + log.trace("pagination page/offset of {}", page); + statement.append(" OFFSET ") + .append(page * size) + .append(";"); return statement.toString(); } @@ -696,93 +652,140 @@ public interface QueryMapper { final List<SelectItem> clauses = ps.getSelectItems(); log.trace("columns referenced in the from-clause: {}", clauses); /* Parse all tables */ - final List<FromItem> tablesOrViews = new ArrayList<>(); - tablesOrViews.add(ps.getFromItem()); - if (ps.getJoins() != null && ps.getJoins().size() > 0) { + final List<FromItem> fromItems = new ArrayList<>(fromItemToFromItems(ps.getFromItem())); + if (ps.getJoins() != null && !ps.getJoins().isEmpty()) { log.trace("query contains join items: {}", ps.getJoins()); for (net.sf.jsqlparser.statement.select.Join j : ps.getJoins()) { if (j.getRightItem() != null) { - tablesOrViews.add(j.getRightItem()); + fromItems.add(j.getRightItem()); } } } - final List<TableColumn> allColumns = database.getTables() - .stream() - .map(Table::getColumns) - .flatMap(List::stream) + final List<ViewColumn> allColumns = Stream.of(database.getViews() + .stream() + .map(View::getColumns) + .flatMap(List::stream), + database.getTables() + .stream() + .map(Table::getColumns) + .flatMap(List::stream) + .map(c -> ViewColumn.builder() + .column(c) + .alias(c.getAlias()) + .ordinalPosition(c.getOrdinalPosition()) + .build()) + ) + .flatMap(i -> i) .toList(); log.trace("columns referenced in the from-clause and join-clause(s): {}", clauses); /* Checking if all tables or views exist */ - log.trace("table(s) or view(s) referenced in the statement: {}", tablesOrViews.stream().map(t -> ((net.sf.jsqlparser.schema.Table) t).getName()).collect(Collectors.toList())); + log.trace("table/view/join referenced in the statement: {}", fromItems.stream().map(this::fromItemToFromItems).flatMap(List::stream).collect(Collectors.toList())); /* Checking if all columns exist */ for (SelectItem clause : clauses) { final SelectExpressionItem item = (SelectExpressionItem) clause; final Column column = (Column) item.getExpression(); - final Optional<net.sf.jsqlparser.schema.Table> optionalTableOrView = tablesOrViews.stream() + final Optional<net.sf.jsqlparser.schema.Table> optional = fromItems.stream() .map(t -> (net.sf.jsqlparser.schema.Table) t) .filter(t -> { if (column.getTable() == null) { /* column does not reference a specific table, so there is only one table */ - final String tableName = ((net.sf.jsqlparser.schema.Table) tablesOrViews.get(0)).getName().replace("`", ""); - return tableOptionalAliasMatches(t, tableName); + final String tableName = ((net.sf.jsqlparser.schema.Table) fromItems.get(0)).getName().replace("`", ""); + return tableMatches(t, tableName); } final String tableName = column.getTable().getName().replace("`", ""); - return tableOptionalAliasMatches(t, tableName); + return tableMatches(t, tableName); }) .findFirst(); - if (optionalTableOrView.isEmpty()) { - log.error("Failed to find table or view with alias {}", column.getTable().getAlias()); - throw new JSQLParserException("Failed to find table or view with alias " + column.getTable().getAlias()); + if (optional.isEmpty()) { + log.error("Failed to find table/view {} (with designator {})", column.getTable().getName(), column.getTable().getAlias()); + throw new JSQLParserException("Failed to find table/view " + column.getTable().getName() + " (with alias " + column.getTable().getAlias() + ")"); } - final Optional<TableColumn> optionalColumn = allColumns.stream() - .filter(c -> c.getInternalName().equals(column.getColumnName().replace("`", ""))) - .filter(c -> columnMatches(c, optionalTableOrView.get().getName().replace("`", ""))) + final String columnName = column.getColumnName().replace("`", ""); + final String tableOrView = optional.get().getName().replace("`", ""); + final List<ViewColumn> filteredColumns = allColumns.stream() + .filter(c -> (c.getAlias() != null && c.getAlias().equals(columnName)) || c.getColumn().getInternalName().equals(columnName)) + .toList(); + final Optional<ViewColumn> optionalColumn = filteredColumns.stream() + .filter(c -> columnMatches(c, tableOrView)) .findFirst(); if (optionalColumn.isEmpty()) { - log.error("Failed to find column with name {} in {}", column.getColumnName(), allColumns.stream().map(TableColumn::getInternalName).toList()); - throw new JSQLParserException("Failed to find column with name " + column.getColumnName() + " in " + allColumns.stream().map(TableColumn::getInternalName).toList()); + log.error("Failed to find column with name {} of table/view {} in {}", columnName, tableOrView, filteredColumns.stream().map(c -> c.getColumn().getTable().getInternalName() + "." + c.getColumn().getInternalName()).toList()); + throw new JSQLParserException("Failed to find column with name " + columnName + " of table/view " + tableOrView); } - final TableColumn aliasColumn = optionalColumn.get(); + final ViewColumn resultColumn = optionalColumn.get(); if (item.getAlias() != null) { - aliasColumn.setAlias(item.getAlias().getName().replace("`", "")); + resultColumn.getColumn().setAlias(item.getAlias().getName().replace("`", "")); } - log.trace("found column with internal name {} and alias {}", aliasColumn.getInternalName(), aliasColumn.getAlias()); - columns.add(aliasColumn); + log.trace("found column with internal name {} and alias {}", resultColumn.getColumn().getInternalName(), resultColumn.getAlias()); + columns.add(resultColumn.getColumn()); } return columns; } - default boolean tableOptionalAliasMatches(net.sf.jsqlparser.schema.Table table, String tableName) { + default List<FromItem> fromItemToFromItems(FromItem data) { + return fromItemToFromItems(data, 0); + } + + default List<FromItem> fromItemToFromItems(FromItem data, Integer level) { + final List<FromItem> fromItems = new LinkedList<>(); + if (data instanceof net.sf.jsqlparser.schema.Table table) { + fromItems.add(data); + log.trace("from-item {} is of type table: level ~> {}", table.getName(), level); + return fromItems; + } + if (data instanceof SubJoin subJoin) { + log.trace("from-item is of type sub-join: level ~> {}", level); + for (Join join : subJoin.getJoinList()) { + fromItems.addAll(fromItemToFromItems(join.getRightItem(), level + 1)); + } + fromItems.addAll(fromItemToFromItems(((SubJoin) data).getLeft(), level + 1)); + return fromItems; + } + log.warn("unknown from-item {}", data); + return null; + } + + default boolean tableMatches(net.sf.jsqlparser.schema.Table table, String otherTableName) { + final String tableName = table.getName() + .trim() + .replace("`", ""); if (table.getAlias() == null) { - /* table is non-aliased */ - final String otherTableName = table.getName() - .trim() - .replace("`", ""); - log.trace("table {} has no alias", otherTableName); - return otherTableName.equals(tableName); + /* table does not have designator */ + log.trace("table {} has no designator", tableName); + return tableName.equals(otherTableName); } - /* has alias */ - final String alias = table.getAlias() + /* has designator */ + final String designator = table.getAlias() .getName() .trim() .replace("`", ""); - log.trace("table {} has alias {}", table.getName(), alias); - return alias.equals(tableName); + log.trace("table {} has designator {}", tableName, designator); + return designator.equals(otherTableName); } @Transactional(readOnly = true) - default boolean columnMatches(TableColumn column, String tableOrView) { - if (column.getTable().getInternalName().equals(tableOrView)) { - /* matches table name */ + default boolean columnMatches(ViewColumn column, String tableOrView) { + if (column.getView() != null && column.getView().getInternalName().equals(tableOrView)) { + log.trace("view {} found in column table", tableOrView); + return true; + } + if (column.getColumn().getTable().getInternalName().equals(tableOrView)) { + log.trace("table {} found in column table", tableOrView); return true; } - if (column.getViews() == null) { + if (column.getColumn().getViews() == null) { + log.trace("table/view {} not found among column views: empty list", tableOrView); return false; } - /* maybe matches one of the views */ - return column.getViews() + /* maybe matches one of the other views */ + final boolean found = column.getColumn() + .getViews() .stream() .anyMatch(v -> v.getInternalName().equals(tableOrView)); + if (!found) { + log.trace("table/view {} not found among column views: {}", tableOrView, column.getColumn().getViews().stream().map(View::getInternalName).toList()); + } + return found; } default PreparedStatement obtainTableMetadataRawQuery(Connection connection, String databaseName, String tableName) throws QueryMalformedException { @@ -796,22 +799,31 @@ public interface QueryMapper { return connection.prepareStatement(statement.toString()); } catch (SQLException e) { log.error("Failed to prepare statement {}: {}", statement, e.getMessage()); - throw new QueryMalformedException("Failed to prepare statement", e); + throw new QueryMalformedException("Failed to prepare statement: " + e.getMessage(), e); } } + default ForeignKeyReference foreignKeyToForeignKeyReference(ForeignKey foreignKey, TableColumn column, + TableColumn referencedColumn) { + return ForeignKeyReference.builder() + .foreignKey(foreignKey) + .column(column) + .referencedColumn(referencedColumn) + .build(); + } + default PreparedStatement databaseToDatabaseConstraintMetadata(Connection connection, String databaseName, String tableName) throws QueryMalformedException { - final StringBuilder statement = new StringBuilder("SELECT tc.`CONSTRAINT_TYPE`, cc.`CONSTRAINT_NAME`, cc.`LEVEL`, cc.`CHECK_CLAUSE`, rc.`UNIQUE_CONSTRAINT_NAME`, rc.`REFERENCED_TABLE_NAME` FROM information_schema.`TABLE_CONSTRAINTS` tc LEFT JOIN information_schema.`CHECK_CONSTRAINTS` cc ON tc.`CONSTRAINT_SCHEMA` = cc.`CONSTRAINT_SCHEMA` AND tc.`TABLE_NAME` = cc.`TABLE_NAME` AND tc.`CONSTRAINT_TYPE` = 'CHECK' LEFT JOIN information_schema.`REFERENTIAL_CONSTRAINTS` rc ON tc.`CONSTRAINT_SCHEMA` = rc.`CONSTRAINT_SCHEMA` AND tc.`TABLE_NAME` = rc.`TABLE_NAME` AND tc.`CONSTRAINT_TYPE` = 'FOREIGN KEY' WHERE tc.`TABLE_SCHEMA` = '") + final StringBuilder statement = new StringBuilder("SELECT tc.`CONSTRAINT_TYPE`, tc.`CONSTRAINT_NAME`, cc.`LEVEL`, cc.`CHECK_CLAUSE`, rc.`UNIQUE_CONSTRAINT_NAME`, kcu.`REFERENCED_TABLE_NAME`, kcu.`COLUMN_NAME`, kcu.`REFERENCED_COLUMN_NAME`FROM information_schema.`TABLE_CONSTRAINTS` tc LEFT JOIN information_schema.`CHECK_CONSTRAINTS` cc ON tc.`CONSTRAINT_SCHEMA` = cc.`CONSTRAINT_SCHEMA` AND tc.`TABLE_NAME` = cc.`TABLE_NAME` AND tc.`CONSTRAINT_TYPE` = 'CHECK' LEFT JOIN information_schema.`REFERENTIAL_CONSTRAINTS` rc ON tc.`CONSTRAINT_SCHEMA` = rc.`CONSTRAINT_SCHEMA` AND tc.`TABLE_NAME` = rc.`TABLE_NAME` AND tc.`CONSTRAINT_TYPE` = 'UNIQUE' LEFT JOIN information_schema.`KEY_COLUMN_USAGE` kcu ON tc.`CONSTRAINT_SCHEMA` = kcu.`CONSTRAINT_SCHEMA` AND tc.`TABLE_NAME` = kcu.`TABLE_NAME` AND (tc.`CONSTRAINT_TYPE` = 'FOREIGN KEY' OR tc.`CONSTRAINT_TYPE` = 'UNIQUE') AND kcu.`CONSTRAINT_NAME` = tc.`CONSTRAINT_NAME` AND LOWER(kcu.`COLUMN_NAME`) != 'row_end' WHERE tc.`TABLE_SCHEMA` = '") .append(databaseName) .append("' AND tc.`TABLE_NAME` = '") .append(tableName) .append("'"); - log.trace("statement={}", statement); + log.trace("mapped obtain table constraint metadata statement {} to prepared statement", statement); try { return connection.prepareStatement(statement.toString()); } catch (SQLException e) { - log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage()); - throw new QueryMalformedException("Failed to prepare statement", e); + log.error("Failed to prepare statement {}: {}", statement, e.getMessage()); + throw new QueryMalformedException("Failed to prepare statement: " + e.getMessage(), e); } } @@ -827,7 +839,7 @@ public interface QueryMapper { return connection.prepareStatement(statement.toString()); } catch (SQLException e) { log.error("Failed to prepare statement {}: {}", statement, e.getMessage()); - throw new QueryMalformedException("Failed to prepare statement", e); + throw new QueryMalformedException("Failed to prepare statement: " + e.getMessage(), e); } } @@ -850,6 +862,11 @@ public interface QueryMapper { if (data == null) { return null; } + /* boolean encoding fix */ + if (column.getColumnType().equals(TableColumnType.TINYINT) && column.getSize() == 1) { + log.debug("column {} is of type tinyint with size {}: map to boolean", column.getInternalName(), column.getSize()); + column.setColumnType(TableColumnType.BOOL); + } switch (column.getColumnType()) { case DATE -> { if (column.getDateFormat() == null) { @@ -888,7 +905,7 @@ public interface QueryMapper { log.trace("mapping {} -> biginteger", data); return new BigInteger(String.valueOf(data)); } - case INT, TINYINT, SMALLINT, MEDIUMINT -> { + case INT, SMALLINT, MEDIUMINT, TINYINT -> { log.trace("mapping {} -> integer", data); return Integer.parseInt(String.valueOf(data)); } @@ -935,37 +952,6 @@ public interface QueryMapper { } } - default String generateInsertFromTemporaryTableSQL(Table table) { - final StringBuilder statement = new StringBuilder("INSERT INTO `") - .append(table.getDatabase().getInternalName()) - .append("`.`") - .append(table.getInternalName()) - .append("` SELECT "); - for (TableColumn tc : table.getColumns()) { - statement.append("`"); - statement.append(tc.getInternalName()).append("`,"); - } - statement.deleteCharAt(statement.length() - 1); - statement.append(" FROM `") - .append(table.getDatabase().getInternalName()) - .append("`.`") - .append(table.getInternalName()) - .append("_temporary`"); - - statement.append(" ON DUPLICATE KEY UPDATE "); - for (TableColumn tc : table.getColumns()) - statement.append("`") - .append(tc.getInternalName()) - .append("`") - .append("=") - .append("VALUES(`") - .append(tc.getInternalName()) - .append("`),"); - statement.deleteCharAt(statement.length() - 1); - statement.append(";"); - return statement.toString(); - } - default void prepareStatementWithColumnTypeObject(PreparedStatement ps, TableColumnType columnType, int idx, Object value) throws SQLException { switch (columnType) { case BLOB, TINYBLOB, MEDIUMBLOB, LONGBLOB: diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/TableMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/TableMapper.java index 999704345ef34d9ad2482a57df73289ebb394f81..b3b734d75e3202c5c6fc31704ec12fd97a0bc5e6 100644 --- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/TableMapper.java +++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/TableMapper.java @@ -85,7 +85,8 @@ public interface TableMapper { ColumnTypeDto columnTypeToColumnTypeDto(TableColumnType data); @Mappings({ - @Mapping(target = "constraints", ignore = true) + @Mapping(target = "constraints", ignore = true), + @Mapping(target = "processedConstraints", expression = "java(false)"), }) Table tableCreateDtoToTable(TableCreateDto data); @@ -124,6 +125,7 @@ public interface TableMapper { default Unique columnNameListToUnique(Table table, List<String> names) throws TableMalformedException { return Unique.builder() .table(table) + .name("UK_" + String.join("_", names)) .columns(columnNameListToTableColumn(table, names)) .build(); } @@ -131,7 +133,7 @@ public interface TableMapper { ReferenceType referenceTypeDtoToReferenceType(ReferenceTypeDto dto); @Transactional(readOnly = true) - default ForeignKey foreignKeyCreateDtoToForeignKey(Table table, ForeignKeyCreateDto data) throws TableMalformedException { + default ForeignKey foreignKeyCreateDtoToForeignKey(Table table, ForeignKeyCreateDto data, Integer index) throws TableMalformedException { final String referencedTableInternalName = nameToInternalName(data.getReferencedTable()); final Optional<Table> optional = table.getDatabase() .getTables() @@ -142,11 +144,13 @@ public interface TableMapper { log.error("Failed to find referenced table with internal name {} in database with id {}", referencedTableInternalName, table.getDatabase().getId()); throw new TableMalformedException("Failed to find referenced table with internal name " + referencedTableInternalName + " in database with id " + table.getDatabase().getId()); } - final ForeignKey.ForeignKeyBuilder builder = ForeignKey.builder() + final ForeignKey foreignKey = ForeignKey.builder() + .name("fk_" + table.getInternalName() + "_" + (index + 1)) .table(table) .onUpdate(referenceTypeDtoToReferenceType(data.getOnUpdate())) .onDelete(referenceTypeDtoToReferenceType(data.getOnDelete())) - .referencedTable(optional.get()); + .referencedTable(optional.get()) + .build(); final List<TableColumn> columns = columnNameListToTableColumn(table, data.getColumns()); final List<TableColumn> referencedColumns = columnNameListToTableColumn(optional.get(), data.getReferencedColumns()); if (columns.isEmpty()) { @@ -158,7 +162,7 @@ public interface TableMapper { throw new TableMalformedException("There have to be equally as many columns and referenced columns in a foreign key"); } final List<ForeignKeyReference> references = new ArrayList<>(); - final ForeignKey foreignKey = builder.references(references).build(); + foreignKey.setReferences(references); for (int i = 0; i < columns.size(); i++) { TableColumn column = columns.get(i); TableColumn referencedColumn = referencedColumns.get(i); @@ -177,44 +181,43 @@ public interface TableMapper { if (data == null) { return null; } - - ForeignKeyDto dto = new ForeignKeyDto( - new ArrayList<>(), - tableToTableBriefDto(data.getReferencedTable()), - new ArrayList<>(), - referenceTypeDtoToReferenceType(data.getOnUpdate()), - referenceTypeDtoToReferenceType(data.getOnDelete()) - ); - + final ForeignKeyDto foreignKey = ForeignKeyDto.builder() + .name(data.getName()) + .columns(new LinkedList<>()) + .referencedColumns(new LinkedList<>()) + .referencedTable(tableToTableBriefDto(data.getReferencedTable())) + .onDelete(referenceTypeDtoToReferenceType(data.getOnDelete())) + .onUpdate(referenceTypeDtoToReferenceType(data.getOnUpdate())) + .build(); for (ForeignKeyReference reference : data.getReferences()) { - dto.getColumns().add(tableColumnToColumnDto(reference.getColumn())); - dto.getReferencedColumns().add(tableColumnToColumnDto(reference.getReferencedColumn())); + foreignKey.getColumns().add(tableColumnToColumnDto(reference.getColumn())); + foreignKey.getReferencedColumns().add(tableColumnToColumnDto(reference.getReferencedColumn())); } - return dto; + return foreignKey; } + @Transactional(readOnly = true) default Constraints constraintsCreateDtoToConstraints(Table table, ConstraintsCreateDto data) throws TableMalformedException { if (data == null) { return null; } - Constraints.ConstraintsBuilder builder = Constraints.builder(); + final Constraints.ConstraintsBuilder builder = Constraints.builder(); if (data.getUniques() != null) { - List<Unique> uniques = new ArrayList<>(); + final List<Unique> uniques = new ArrayList<>(); for (List<String> columns : data.getUniques()) { uniques.add(columnNameListToUnique(table, columns)); } builder.uniques(uniques); } if (data.getForeignKeys() != null) { - List<ForeignKey> foreignKeys = new ArrayList<>(); - for (ForeignKeyCreateDto foreignKeyData : data.getForeignKeys()) { - foreignKeys.add(foreignKeyCreateDtoToForeignKey(table, foreignKeyData)); + final List<ForeignKey> foreignKeys = new ArrayList<>(); + for (int i = 0; i < data.getForeignKeys().size(); i++) { + foreignKeys.add(foreignKeyCreateDtoToForeignKey(table, data.getForeignKeys().get(i), i)); } builder.foreignKeys(foreignKeys); } - return builder.build(); } @@ -481,7 +484,7 @@ public interface TableMapper { } default PreparedStatement tableToCreateHistoryViewRawQuery(Connection connection, Table data) throws QueryMalformedException { - final StringBuilder view = new StringBuilder("CREATE VIEW `hs_") + final StringBuilder view = new StringBuilder("CREATE VIEW IF NOT EXISTS `hs_") .append(data.getInternalName()) .append("` AS SELECT * FROM (SELECT ROW_START AS inserted_at, IF(ROW_END > NOW(), NULL, ROW_END) AS deleted_at, COUNT(*) as total FROM `") .append(data.getInternalName()) @@ -588,7 +591,10 @@ public interface TableMapper { .name(resultSet.getString(10)) .internalName(resultSet.getString(10)) .build(); - if (resultSet.getString(5) != null) { + /* fix boolean and set size for others */ + if (resultSet.getString(8).equalsIgnoreCase("tinyint(1)")) { + column.setColumnType(TableColumnType.BOOL); + } else if (resultSet.getString(5) != null) { column.setSize(resultSet.getLong(5)); } else if (resultSet.getString(6) != null) { column.setSize(resultSet.getLong(6)); diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/ViewMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/ViewMapper.java index 272275fec6a2ac44ff5748585d0c365510cb5da6..c63098a06f964a2cc28a15f8f28910809ad5d0c9 100644 --- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/ViewMapper.java +++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/ViewMapper.java @@ -4,16 +4,20 @@ import at.tuwien.api.database.ViewBriefDto; import at.tuwien.api.database.ViewCreateDto; import at.tuwien.api.database.ViewDto; import at.tuwien.entities.database.View; +import at.tuwien.entities.database.ViewColumn; +import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.exception.QueryMalformedException; import org.mapstruct.Mapper; import org.mapstruct.Mapping; import org.mapstruct.Mappings; import org.mapstruct.Named; +import org.springframework.transaction.annotation.Transactional; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.text.Normalizer; +import java.util.List; import java.util.Locale; import java.util.regex.Pattern; @@ -45,15 +49,34 @@ public interface ViewMapper { ViewBriefDto viewToViewBriefDto(View data); + @Transactional(readOnly = true) + default TableColumn viewColumnToTableColumn(ViewColumn data) { + return data.getColumn() + .toBuilder() + .alias(data.getAlias()) + .build(); + } + + default List<ViewColumn> tableColumnsToViewColumns(View view, List<TableColumn> data) { + final int[] idx = new int[]{0}; + return data.stream() + .map(c -> ViewColumn.builder() + .ordinalPosition(idx[0]++) + .column(c) + .view(view) + .alias(c.getAlias()) + .build()) + .toList(); + } default PreparedStatement viewToSelectAll(Connection connection, View view, Long page, Long size) throws QueryMalformedException { - log.debug("mapping view query, view.query={}", view.getQuery()); + log.debug("mapping view query, view.query={}, page={}, size={}", view.getQuery(), page, size); final StringBuilder statement = new StringBuilder("SELECT "); final int[] idx = new int[]{0}; view.getColumns() .forEach(c -> statement.append(idx[0]++ > 0 ? "," : "") .append("`") - .append(c.getInternalName()) + .append(c.getAlias() != null ? c.getAlias() : c.getColumn().getInternalName()) .append("`")); statement.append(" FROM `") .append(view.getInternalName()) diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableColumnEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableColumnEndpoint.java index 47193508ba32c0cd0ee33091b5860f7c3380b5c7..acd980cc2dfb256129414a11278ef0feea5c1489 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableColumnEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableColumnEndpoint.java @@ -6,7 +6,7 @@ import at.tuwien.api.error.ApiErrorDto; import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.exception.*; import at.tuwien.mapper.TableMapper; -import at.tuwien.service.TableService; +import at.tuwien.service.TableColumnService; import at.tuwien.utils.PrincipalUtil; import at.tuwien.utils.UserUtil; import at.tuwien.validation.EndpointValidator; @@ -35,14 +35,15 @@ import java.security.Principal; public class TableColumnEndpoint { private final TableMapper tableMapper; - private final TableService tableService; private final EndpointValidator endpointValidator; + private final TableColumnService tableColumnService; @Autowired - public TableColumnEndpoint(TableMapper tableMapper, TableService tableService, EndpointValidator endpointValidator) { + public TableColumnEndpoint(TableMapper tableMapper, EndpointValidator endpointValidator, + TableColumnService tableColumnService) { this.tableMapper = tableMapper; - this.tableService = tableService; this.endpointValidator = endpointValidator; + this.tableColumnService = tableColumnService; } @PutMapping @@ -76,8 +77,7 @@ public class TableColumnEndpoint { @NotNull @PathVariable("tableId") Long tableId, @NotNull @PathVariable("columnId") Long columnId, @NotNull @Valid @RequestBody ColumnSemanticsUpdateDto updateDto, - @NotNull Principal principal, - @NotNull @RequestHeader("Authorization") String authorization) + @NotNull Principal principal) throws TableNotFoundException, TableMalformedException, DatabaseNotFoundException, NotAllowedException, AccessDeniedException { log.debug("endpoint update table, id={}, tableId={}, columnId={}, {}", id, tableId, columnId, PrincipalUtil.formatForDebug(principal)); @@ -85,11 +85,12 @@ public class TableColumnEndpoint { endpointValidator.validateOnlyAccess(id, principal, true); endpointValidator.validateOnlyOwnerOrWriteAll(id, tableId, principal); } - final TableColumn column = tableService.update(id, tableId, columnId, updateDto, authorization); + final TableColumn column = tableColumnService.update(id, tableId, columnId, updateDto); log.info("Updated table semantics of table with id {} and database with id {}", tableId, id); - final ColumnDto dto = tableMapper.tableColumnToColumnDto(column); + final ColumnDto columnDto = tableMapper.tableColumnToColumnDto(column); + log.trace("find table data resulted in column {}", columnDto); return ResponseEntity.accepted() - .body(dto); + .body(columnDto); } } diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java index fe12e3aeff6d8e7224e1b57b5f54436fc2ac1f7b..a06d7987f0a46840ada94bc2d9ab9138b37a9bc8 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java @@ -14,7 +14,6 @@ import at.tuwien.utils.PrincipalUtil; import at.tuwien.utils.UserUtil; import at.tuwien.validation.EndpointValidator; import io.micrometer.observation.annotation.Observed; -import io.swagger.v3.oas.annotations.ExternalDocumentation; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; @@ -174,6 +173,14 @@ public class TableDataEndpoint { log.debug("endpoint insert data from csv, databaseId={}, tableId={}, data={}, {}", databaseId, tableId, data, PrincipalUtil.formatForDebug(principal)); /* check */ endpointValidator.validateOnlyWriteOwnOrWriteAllAccess(databaseId, tableId, principal); + if (data.getNullElement() == null) { + log.debug("null element not present, default to empty string"); + data.setNullElement(""); + } + if (data.getLineTermination() == null) { + log.debug("line termination not present, default to \\r\\n"); + data.setLineTermination("\r\n"); + } /* insert */ queryService.insert(databaseId, tableId, data, principal); return ResponseEntity.accepted() @@ -229,6 +236,15 @@ public class TableDataEndpoint { log.error("Failed to view table data: database with id {} is private and user has no authority", databaseId); throw new NotAllowedException("Failed to view table data: database with id " + databaseId + " is private and user has no authority"); } + /* default */ + if (page == null) { + log.trace("page is null: default to 0"); + page = 0L; + } + if (size == null) { + log.trace("size is null: default to 10"); + size = 10L; + } /* find */ final QueryResultDto response = queryService.tableFindAll(databaseId, tableId, timestamp, page, size, principal); log.trace("find table data resulted in result {}", response); diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java index f3a1b76c06de61b3867fac1c1415f5034d179b21..a4084950ac3c5a76a94fcfa46cfa3287b8ea1664 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java @@ -294,6 +294,15 @@ public class ViewEndpoint { throw new NotAllowedException("Failed to view data of private view: role missing"); } } + /* default */ + if (page == null) { + log.trace("page is null: default to 0"); + page = 0L; + } + if (size == null) { + log.trace("size is null: default to 10"); + size = 10L; + } /* find */ log.debug("find view data for database with id {}", databaseId); final View view = viewService.findById(databaseId, viewId, principal); diff --git a/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml b/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml index 1ae0d498b22b598e1d1306fdb48cf681cc778b64..d759af61d3e67ffa18c7fe332fa7f5dd286b1209 100644 --- a/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml +++ b/dbrepo-metadata-service/rest-service/src/main/resources/application-local.yml @@ -40,7 +40,7 @@ spring: loadbalancer.ribbon.enabled: false management.endpoints.web.exposure.include: health,info,prometheus server: - port: 19099 + port: 9099 logging: pattern.console: "%d %highlight(%-5level) %msg%n" level: diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java index fab3add8f6bc3b103fd9b59d5cbd162ae5903b80..038089a720f04c23715f8931b16a605c8bf933a1 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java @@ -327,7 +327,7 @@ public class MariaDbConfig { } public static ColumnTypeDto typetoColumnTypeDto(String data) throws Exception { - if (data.toUpperCase().startsWith("TINYINT(1)")) { + if (data.equalsIgnoreCase("TINYINT(1)")) { /* boolean in MySQL */ return ColumnTypeDto.BOOL; } diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableColumnEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableColumnEndpointUnitTest.java index 5ed2d91817748146788778f2a9a538a83bb9d312..697fd5dff2816e725e111018977c461806272db1 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableColumnEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableColumnEndpointUnitTest.java @@ -12,6 +12,7 @@ import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.exception.*; import at.tuwien.service.AccessService; import at.tuwien.service.DatabaseService; +import at.tuwien.service.TableColumnService; import at.tuwien.service.TableService; import lombok.extern.log4j.Log4j2; import org.junit.jupiter.api.Test; @@ -50,6 +51,9 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest { @MockBean private TableService tableService; + @MockBean + private TableColumnService tableColumnService; + @Autowired private TableColumnEndpoint tableColumnEndpoint; @@ -95,8 +99,7 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest { @Test @WithMockUser(username = USER_1_USERNAME, authorities = {"modify-table-column-semantics"}) public void update_publicHasRoleHasOwnWriteAccess_succeeds() throws TableNotFoundException, NotAllowedException, - TableMalformedException, DatabaseNotFoundException, ContainerNotFoundException, - SemanticEntityPersistException, SemanticEntityNotFoundException, QueryMalformedException, at.tuwien.exception.AccessDeniedException { + TableMalformedException, DatabaseNotFoundException, at.tuwien.exception.AccessDeniedException { final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder() .unitUri(UNIT_MILLIMETRE_URI) .build(); @@ -148,8 +151,7 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest { @WithMockUser(username = USER_2_USERNAME, authorities = {"modify-table-column-semantics"}) public void update_publicHasRoleForeignHasAllWriteAccess_succeeds() throws TableNotFoundException, NotAllowedException, TableMalformedException, DatabaseNotFoundException, - ContainerNotFoundException, SemanticEntityPersistException, SemanticEntityNotFoundException, - QueryMalformedException, at.tuwien.exception.AccessDeniedException { + at.tuwien.exception.AccessDeniedException { final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder() .unitUri(UNIT_MILLIMETRE_URI) .build(); @@ -204,9 +206,7 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest { @Test @WithMockUser(username = USER_1_USERNAME, authorities = {"modify-table-column-semantics"}) public void update_privateHasRoleHasOwnWriteAccess_succeeds() throws TableNotFoundException, NotAllowedException, - TableMalformedException, DatabaseNotFoundException, ContainerNotFoundException, - SemanticEntityPersistException, SemanticEntityNotFoundException, QueryMalformedException, - at.tuwien.exception.AccessDeniedException { + TableMalformedException, DatabaseNotFoundException, at.tuwien.exception.AccessDeniedException { final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder() .unitUri(UNIT_MILLIMETRE_URI) .build(); @@ -258,8 +258,7 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest { @WithMockUser(username = USER_2_USERNAME, authorities = {"modify-table-column-semantics"}) public void update_privateHasRoleForeignHasAllWriteAccess_succeeds() throws TableNotFoundException, NotAllowedException, TableMalformedException, DatabaseNotFoundException, - ContainerNotFoundException, SemanticEntityPersistException, SemanticEntityNotFoundException, - QueryMalformedException, at.tuwien.exception.AccessDeniedException { + at.tuwien.exception.AccessDeniedException { final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder() .unitUri(UNIT_MILLIMETRE_URI) .build(); @@ -277,8 +276,7 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest { ColumnSemanticsUpdateDto data, UUID userId, Principal principal, DatabaseAccess access) throws DatabaseNotFoundException, NotAllowedException, TableNotFoundException, TableMalformedException, - ContainerNotFoundException, SemanticEntityPersistException, SemanticEntityNotFoundException, - QueryMalformedException, at.tuwien.exception.AccessDeniedException { + at.tuwien.exception.AccessDeniedException { /* mock */ if (database != null) { @@ -292,12 +290,12 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest { if (table != null) { when(tableService.find(databaseId, tableId)) .thenReturn(table); - when(tableService.update(databaseId, tableId, columnId, data, "abc")) + when(tableColumnService.update(databaseId, tableId, columnId, data)) .thenReturn(column); } else { doThrow(TableNotFoundException.class) - .when(tableService) - .update(databaseId, tableId, columnId, data, "abc"); + .when(tableColumnService) + .update(databaseId, tableId, columnId, data); doThrow(TableNotFoundException.class) .when(tableService) .find(databaseId, tableId); @@ -312,6 +310,6 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest { } /* test */ - return tableColumnEndpoint.update(databaseId, tableId, columnId, data, principal, "abc"); + return tableColumnEndpoint.update(databaseId, tableId, columnId, data, principal); } } diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java index 933d4ba78aefde918ecbbaa7f4f63fb5c5249871..a866e8240f6c71e5e3e1258d696eaf0986f667f6 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java @@ -3,6 +3,7 @@ package at.tuwien.endpoints; import at.tuwien.BaseUnitTest; import at.tuwien.SortType; import at.tuwien.annotations.MockAmqp; +import at.tuwien.annotations.MockListeners; import at.tuwien.annotations.MockOpensearch; import at.tuwien.api.database.query.ImportDto; import at.tuwien.api.database.query.QueryResultDto; @@ -36,6 +37,7 @@ import java.util.UUID; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.when; @Log4j2 @@ -43,6 +45,7 @@ import static org.mockito.Mockito.when; @ExtendWith(SpringExtension.class) @MockAmqp @MockOpensearch +@MockListeners public class TableDataEndpointUnitTest extends BaseUnitTest { @MockBean @@ -469,7 +472,7 @@ public class TableDataEndpointUnitTest extends BaseUnitTest { .thenReturn(table); when(accessService.find(databaseId, userId)) .thenReturn(access); - when(queryService.tableFindAll(databaseId, tableId, timestamp, page, size, principal)) + when(queryService.tableFindAll(eq(databaseId), eq(tableId), eq(timestamp), anyLong(), anyLong(), eq(principal))) .thenReturn(QUERY_1_RESULT_DTO); /* test */ diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java index 94a97d3643140ef3de727720821c1e1b37f94bb0..26a9b506e5aa3809c26646fe5f4abbc27ac36527 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java @@ -590,7 +590,7 @@ public class PrometheusEndpointMvcTest extends BaseUnitTest { /* mock */ try { - tableColumnEndpoint.update(DATABASE_1_ID, TABLE_1_ID, TABLE_1_COLUMNS.get(3).getId(), request, USER_1_PRINCIPAL, "s3cr3t"); + tableColumnEndpoint.update(DATABASE_1_ID, TABLE_1_ID, TABLE_1_COLUMNS.get(3).getId(), request, USER_1_PRINCIPAL); } catch (Exception e) { /* ignore */ } diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/repository/DatabaseRepositoryIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/repository/DatabaseRepositoryIntegrationTest.java index 0e7473b849b560f28d93feb6705d26c366ab0125..537f5f9882c12607d43894518575e33c1e011572 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/repository/DatabaseRepositoryIntegrationTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/repository/DatabaseRepositoryIntegrationTest.java @@ -50,6 +50,12 @@ public class DatabaseRepositoryIntegrationTest extends BaseUnitTest { TABLE_5.setColumns(TABLE_5_COLUMNS); TABLE_6.setColumns(TABLE_6_COLUMNS); TABLE_7.setColumns(TABLE_7_COLUMNS); + DATABASE_1.setAccesses(List.of()); + DATABASE_2.setAccesses(List.of()); + VIEW_1.setColumns(VIEW_1_COLUMNS); + VIEW_2.setColumns(VIEW_2_COLUMNS); + VIEW_3.setColumns(VIEW_3_COLUMNS); + VIEW_4.setColumns(VIEW_4_COLUMNS); /* metadata database */ imageRepository.save(IMAGE_1); licenseRepository.save(LICENSE_1); diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java index c51f14874500e4763242c9e666d0321ea0dbd48c..38fd8f645f66a1e9a866788b7b653cb9eb1729cf 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java @@ -13,6 +13,10 @@ import at.tuwien.entities.database.View; import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.entities.database.table.columns.TableColumnType; +import at.tuwien.entities.database.table.constraints.Constraints; +import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKey; +import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKeyReference; +import at.tuwien.entities.database.table.constraints.unique.Unique; import at.tuwien.entities.user.User; import at.tuwien.exception.*; import at.tuwien.repository.mdb.*; @@ -341,11 +345,11 @@ public class DatabaseServiceIntegrationTest extends BaseUnitTest { } @Test - public void obtainMetadata_tableWithoutVersioning_succeeds() throws QueryMalformedException, + public void obtainTablesMetadata_tableWithoutVersioning_succeeds() throws QueryMalformedException, DatabaseNotFoundException, ColumnParseException { /* test */ - final Database response = databaseService.obtainMetadata(DATABASE_1_ID); + final Database response = databaseService.obtainTablesMetadata(DATABASE_1_ID); final List<Table> tables = response.getTables(); assertEquals(7, tables.size()); final Optional<Table> optional3 = tables.stream().filter(t -> t.getInternalName().equals("weather_aut_without_versioning")).findFirst(); @@ -360,11 +364,11 @@ public class DatabaseServiceIntegrationTest extends BaseUnitTest { } @Test - public void obtainMetadata_tableWithVersioning_succeeds() throws QueryMalformedException, DatabaseNotFoundException, + public void obtainTablesMetadata_tableWithVersioning_succeeds() throws QueryMalformedException, DatabaseNotFoundException, ColumnParseException { /* test */ - final Database response = databaseService.obtainMetadata(DATABASE_1_ID); + final Database response = databaseService.obtainTablesMetadata(DATABASE_1_ID); final List<Table> tables = response.getTables(); assertEquals(7, tables.size()); final Optional<Table> optional4 = tables.stream().filter(t -> t.getInternalName().equals("weather_aut")).findFirst(); @@ -380,11 +384,14 @@ public class DatabaseServiceIntegrationTest extends BaseUnitTest { } @Test - public void obtainMetadata_view_succeeds() throws QueryMalformedException, DatabaseNotFoundException, + public void obtainViewsMetadata_view_succeeds() throws QueryMalformedException, DatabaseNotFoundException, ColumnParseException { + /* mock */ + databaseService.obtainTablesMetadata(DATABASE_1_ID); /* weather_aut is not yet in metadata-db */ + /* test */ - final Database response = databaseService.obtainMetadata(DATABASE_1_ID); + final Database response = databaseService.obtainViewsMetadata(DATABASE_1_ID); final List<Table> tables = response.getTables(); assertEquals(7, tables.size()); final List<View> views = response.getViews(); @@ -400,14 +407,65 @@ public class DatabaseServiceIntegrationTest extends BaseUnitTest { assertEquals(DATABASE_1_OWNER, view1.getCreatedBy()); assertNotNull(view1.getQuery()); assertNotNull(view1.getQueryHash()); - assertColumn(view1.getColumns().get(0), 0, "id", TableColumnType.BIGINT, null, false, true, true); - assertColumn(view1.getColumns().get(1), 1, "date", TableColumnType.DATE, null, false, false, false); + assertColumn(view1.getColumns().get(0).getColumn(), 0, "id", TableColumnType.BIGINT, null, false, true, true); + assertColumn(view1.getColumns().get(1).getColumn(), 1, "date", TableColumnType.DATE, null, false, false, false); + } + + @Test + public void obtainConstraints_inlineConstraints_succeeds() throws QueryMalformedException, + DatabaseNotFoundException, TableMalformedException, SQLException, ColumnParseException { + + /* test */ + generic_obtainConstraints("CREATE TABLE foreigner (id BIGINT PRIMARY KEY NOT NULL, weather_id BIGINT REFERENCES weather_aus (id), qty INT CHECK (qty > 0), firstname VARCHAR(255) UNIQUE) WITH SYSTEM VERSIONING;"); + } + + @Test + public void obtainConstraints_complexConstraints_succeeds() throws QueryMalformedException, + DatabaseNotFoundException, TableMalformedException, SQLException, ColumnParseException { + + /* test */ + generic_obtainConstraints("CREATE TABLE foreigner (id BIGINT NOT NULL, weather_id BIGINT NOT NULL, qty INT NOT NULL, firstname VARCHAR(255) NOT NULL, PRIMARY KEY (id), UNIQUE (firstname), FOREIGN KEY (weather_id) REFERENCES weather_aus (id), CONSTRAINT pos_qty CHECK (qty > 0)) WITH SYSTEM VERSIONING;"); } /* ################################################################################################### */ /* ## GENERIC TEST CASES ## */ /* ################################################################################################### */ + protected void generic_obtainConstraints(String sql) throws QueryMalformedException, DatabaseNotFoundException, + TableMalformedException, SQLException, ColumnParseException { + + /* mock */ + MariaDbConfig.execute(DATABASE_1, sql); + databaseService.obtainTablesMetadata(DATABASE_1_ID); + + /* test */ + final Database response = databaseService.obtainConstraints(DATABASE_1_ID); + final List<Table> tables = response.getTables(); + assertEquals(8, tables.size()); + final Optional<Table> optional8 = tables.stream().filter(t -> t.getInternalName().equals("foreigner")).findFirst(); + assertTrue(optional8.isPresent()); + final Table table8 = optional8.get(); + assertNotNull(table8.getConstraints()); + final Constraints constraints8 = table8.getConstraints(); + assertNotNull(constraints8.getUniques()); + assertEquals(1, constraints8.getUniques().size()); + final Unique unique0 = constraints8.getUniques().get(0); + assertEquals("foreigner", unique0.getTable().getInternalName()); + assertEquals(1, unique0.getColumns().size()); + assertEquals("firstname", unique0.getColumns().get(0).getInternalName()); + assertNotNull(constraints8.getChecks()); + assertEquals(1, constraints8.getChecks().size()); + assertNotNull(constraints8.getForeignKeys()); + assertEquals(1, constraints8.getForeignKeys().size()); + final ForeignKey foreignKey0 = constraints8.getForeignKeys().get(0); + assertEquals("foreigner", foreignKey0.getTable().getInternalName()); + assertEquals("weather_aus", foreignKey0.getReferencedTable().getInternalName()); + assertEquals(1, foreignKey0.getReferences().size()); + final ForeignKeyReference foreignKeyReference0 = foreignKey0.getReferences().get(0); + assertEquals("weather_id", foreignKeyReference0.getColumn().getInternalName()); + assertEquals("id", foreignKeyReference0.getReferencedColumn().getInternalName()); + } + protected void generic_insert(String query, Long assertQueryId) throws SQLException, QueryMalformedException { /* mock */ diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceIntegrationTest.java index 7ec7d0a78475e561b177dbc16d911a4d9d7a8bd5..185c7cda6191f756a84efdd959b70d4f4f32cd9f 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceIntegrationTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceIntegrationTest.java @@ -56,6 +56,9 @@ public class MetadataServiceIntegrationTest extends BaseUnitTest { TABLE_2.setColumns(TABLE_2_COLUMNS); TABLE_3.setColumns(TABLE_3_COLUMNS); TABLE_4.setColumns(TABLE_4_COLUMNS); + VIEW_1.setColumns(VIEW_1_COLUMNS); + VIEW_2.setColumns(VIEW_2_COLUMNS); + VIEW_3.setColumns(VIEW_3_COLUMNS); /* metadata database */ imageRepository.save(IMAGE_1); userRepository.save(USER_1); diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java index 77af8cb2fcd0e400b5ab5c5c0d60adad1f05993b..560fd4e89d6a8e5210a0bd0043fbe38dc7f6f52e 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java @@ -88,6 +88,9 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { @Autowired private LicenseRepository licenseRepository; + @Autowired + private DatabaseService databaseService; + @MockBean private DataDbSidecarGateway dataDbSidecarGateway; @@ -105,7 +108,8 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { } @BeforeEach - public void beforeEach() throws SQLException { + public void beforeEach() throws SQLException, DatabaseUnchangedException, QueryMalformedException, + ColumnParseException, DatabaseNotFoundException, TableMalformedException { TABLE_1.setColumns(TABLE_1_COLUMNS); TABLE_2.setColumns(TABLE_2_COLUMNS); TABLE_3.setColumns(TABLE_3_COLUMNS); @@ -115,6 +119,10 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { TABLE_7.setColumns(TABLE_7_COLUMNS); DATABASE_1.setAccesses(List.of()); DATABASE_2.setAccesses(List.of()); + VIEW_1.setColumns(VIEW_1_COLUMNS); + VIEW_2.setColumns(VIEW_2_COLUMNS); + VIEW_3.setColumns(VIEW_3_COLUMNS); + VIEW_4.setColumns(VIEW_4_COLUMNS); /* metadata database */ imageRepository.save(IMAGE_1); licenseRepository.save(LICENSE_1); @@ -125,6 +133,12 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { MariaDbConfig.dropAllDatabases(CONTAINER_1); MariaDbConfig.createInitDatabase(CONTAINER_1, DATABASE_1); MariaDbConfig.createInitDatabase(CONTAINER_1, DATABASE_2); + databaseService.obtainTablesMetadata(DATABASE_1_ID); + databaseService.obtainTablesMetadata(DATABASE_2_ID); + databaseService.obtainConstraints(DATABASE_1_ID); + databaseService.obtainConstraints(DATABASE_2_ID); + databaseService.obtainViewsMetadata(DATABASE_1_ID); + databaseService.obtainViewsMetadata(DATABASE_2_ID); } @Test @@ -134,7 +148,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { /* test */ final QueryResultDto result = queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, Instant.now(), - null, null, USER_1_PRINCIPAL); + 0L, 10L, USER_1_PRINCIPAL); assertEquals(3, result.getResult().size()); assertEquals(BigInteger.valueOf(1L), result.getResult().get(0).get(TABLE_1_COLUMNS.get(0).getInternalName())); assertEquals(toInstant("2008-12-01"), result.getResult().get(0).get(TABLE_1_COLUMNS.get(1).getInternalName())); @@ -195,6 +209,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { .nullElement("NA") .separator(';') .location(filename) + .lineTermination("\r\n") .build(); /* mock */ @@ -319,7 +334,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException { /* test */ - queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, null, null, null, USER_1_PRINCIPAL); + queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, null, 0L, 10L, USER_1_PRINCIPAL); } @Test @@ -328,8 +343,8 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { final Instant timestamp = DATABASE_1_CREATED.minus(1, ChronoUnit.SECONDS); /* test */ - queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, timestamp, null, null, USER_1_PRINCIPAL); - queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, timestamp, null, null, USER_1_PRINCIPAL); + queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, timestamp, 0L, 10L, USER_1_PRINCIPAL); + queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, timestamp, 0L, 10L, USER_1_PRINCIPAL); } @Test @@ -452,22 +467,31 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { assertEquals(9L, response.getResultNumber()); assertNotNull(response.getResult()); final List<Map<String, Object>> result = response.getResult(); + assertEquals(2, result.get(0).keySet().size()); assertEquals("Albury", result.get(0).get("a")); assertEquals("Albury", result.get(0).get("location")); + assertEquals(2, result.get(1).keySet().size()); assertEquals("Albury", result.get(1).get("a")); assertEquals("Albury", result.get(1).get("location")); + assertEquals(2, result.get(2).keySet().size()); assertEquals("Albury", result.get(2).get("a")); assertEquals("Albury", result.get(2).get("location")); + assertEquals(2, result.get(3).keySet().size()); assertEquals("Albury", result.get(3).get("a")); assertEquals("Sydney", result.get(3).get("location")); + assertEquals(2, result.get(4).keySet().size()); assertEquals("Albury", result.get(4).get("a")); assertEquals("Sydney", result.get(4).get("location")); + assertEquals(2, result.get(5).keySet().size()); assertEquals("Albury", result.get(5).get("a")); assertEquals("Sydney", result.get(5).get("location")); + assertEquals(2, result.get(6).keySet().size()); assertEquals("Albury", result.get(6).get("a")); assertEquals("Vienna", result.get(6).get("location")); + assertEquals(2, result.get(7).keySet().size()); assertEquals("Albury", result.get(7).get("a")); assertEquals("Vienna", result.get(7).get("location")); + assertEquals(2, result.get(8).keySet().size()); assertEquals("Albury", result.get(8).get("a")); assertEquals("Vienna", result.get(8).get("location")); } @@ -523,18 +547,18 @@ public class QueryServiceIntegrationTest extends BaseUnitTest { /* ordering */ final String[] keys = result.get(0).keySet().toArray(new String[0]); assertEquals("date", keys[0]); - assertEquals("rainfall", keys[1]); - assertEquals("location", keys[2]); + assertEquals("loc", keys[1]); + assertEquals("rainfall", keys[2]); assertEquals("mintemp", keys[3]); /* values */ assertEquals(0.6, result.get(0).get("rainfall")); - assertEquals("Albury", result.get(0).get("location")); + assertEquals("Albury", result.get(0).get("loc")); assertEquals(13.4, result.get(0).get("mintemp")); assertEquals(0.0, result.get(1).get("rainfall")); - assertEquals("Albury", result.get(1).get("location")); + assertEquals("Albury", result.get(1).get("loc")); assertEquals(7.4, result.get(1).get("mintemp")); assertEquals(0.0, result.get(2).get("rainfall")); - assertEquals("Albury", result.get(2).get("location")); + assertEquals("Albury", result.get(2).get("loc")); assertEquals(12.9, result.get(2).get("mintemp")); } diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableColumnServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableColumnServiceIntegrationTest.java new file mode 100644 index 0000000000000000000000000000000000000000..2391a5229007710d4d175448707ecd022190004e --- /dev/null +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableColumnServiceIntegrationTest.java @@ -0,0 +1,107 @@ +package at.tuwien.service; + +import at.tuwien.BaseUnitTest; +import at.tuwien.annotations.MockAmqp; +import at.tuwien.annotations.MockListeners; +import at.tuwien.annotations.MockOpensearch; +import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto; +import at.tuwien.config.MariaDbConfig; +import at.tuwien.config.MariaDbContainerConfig; +import at.tuwien.entities.database.table.columns.TableColumn; +import at.tuwien.entities.database.table.columns.TableColumnConcept; +import at.tuwien.exception.*; +import at.tuwien.repository.mdb.*; +import lombok.extern.log4j.Log4j2; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.sql.SQLException; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +@Log4j2 +@Testcontainers +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) +@EnableAutoConfiguration(exclude = RabbitAutoConfiguration.class) +@SpringBootTest +@ExtendWith(SpringExtension.class) +@MockAmqp +@MockListeners +@MockOpensearch +public class TableColumnServiceIntegrationTest extends BaseUnitTest { + + @Autowired + private ImageRepository imageRepository; + + @Autowired + private ContainerRepository containerRepository; + + @Autowired + private DatabaseRepository databaseRepository; + + @Autowired + private LicenseRepository licenseRepository; + + @Autowired + private UserRepository userRepository; + + @Autowired + private TableColumnService tableColumnService; + + @Container + private static MariaDBContainer<?> mariaDBContainer = MariaDbContainerConfig.getContainer(); + + @BeforeEach + public void beforeEach() throws SQLException { + TABLE_1.setColumns(TABLE_1_COLUMNS); + TABLE_1_FOREIGN_KEY_1.setReferences(List.of(TABLE_1_FOREIGN_KEY_REFERENCE)); + TABLE_1.setConstraints(TABLE_1_CONSTRAINTS); + TABLE_2.setColumns(TABLE_2_COLUMNS); + TABLE_2.setConstraints(TABLE_2_CONSTRAINTS); + TABLE_3.setColumns(TABLE_3_COLUMNS); + TABLE_3.setConstraints(TABLE_3_CONSTRAINTS); + TABLE_4.setColumns(TABLE_4_COLUMNS); + DATABASE_1.setAccesses(List.of()); + TABLE_1.setDatabase(DATABASE_1); + TABLE_2.setDatabase(DATABASE_1); + TABLE_3.setDatabase(DATABASE_1); + TABLE_4.setDatabase(DATABASE_1); + /* metadata database */ + imageRepository.save(IMAGE_1); + licenseRepository.save(LICENSE_1); + userRepository.saveAll(List.of(USER_1, USER_2)); + containerRepository.save(CONTAINER_1); + databaseRepository.save(DATABASE_1); + /* data stuff */ + MariaDbConfig.dropAllDatabases(CONTAINER_1); + MariaDbConfig.createInitDatabase(CONTAINER_1, DATABASE_1); + } + + @Test + @Transactional + public void update_succeeds() throws TableNotFoundException, TableMalformedException, DatabaseNotFoundException { + final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder() + .conceptUri(COLUMN_CONCEPT_PRECIPITATION_URI) + .build(); + + /* test */ + final TableColumn response = tableColumnService.update(DATABASE_1_ID, TABLE_1_ID, TABLE_1_COLUMNS.get(0).getId(), + request); + assertNotNull(response.getConcept()); + final TableColumnConcept concept = response.getConcept(); + assertEquals(COLUMN_CONCEPT_PRECIPITATION_URI, concept.getUri()); + } + +} diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationReadTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationReadTest.java index 4220af3f9ccb5515e5f1c79c0ad29de411dc00fe..c5a2400daf731d5d7e673b18c8f425fad67b44ce 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationReadTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationReadTest.java @@ -98,7 +98,7 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest { } @Test - public void findById_succeeds() throws TableNotFoundException, DatabaseNotFoundException{ + public void findById_succeeds() throws TableNotFoundException, DatabaseNotFoundException { /* test */ final Table response = tableService.find(DATABASE_1_ID, TABLE_1_ID); @@ -126,8 +126,8 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest { } @Test - public void findHistory_anonymous_succeeds() throws UserNotFoundException, TableNotFoundException, - QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException { + public void findHistory_anonymous_succeeds() throws TableNotFoundException, QueryStoreException, + QueryMalformedException, DatabaseNotFoundException { /* test */ final List<TableHistoryDto> response = tableService.findHistory(DATABASE_1_ID, TABLE_1_ID, null); @@ -138,8 +138,8 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest { @Test @WithAnonymousUser - public void findHistory_anonymous2_succeeds() throws UserNotFoundException, TableNotFoundException, - QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException { + public void findHistory_anonymous2_succeeds()throws TableNotFoundException, QueryStoreException, + QueryMalformedException, DatabaseNotFoundException { /* test */ final List<TableHistoryDto> response = tableService.findHistory(DATABASE_1_ID, TABLE_1_ID, null); @@ -150,8 +150,8 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest { @Test @WithMockUser(username = USER_1_USERNAME, roles = {"RESEARCHER"}) - public void findHistory_researcher_succeeds() throws UserNotFoundException, TableNotFoundException, - QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException { + public void findHistory_researcher_succeeds() throws TableNotFoundException, QueryStoreException, + QueryMalformedException, DatabaseNotFoundException { /* test */ final List<TableHistoryDto> response = tableService.findHistory(DATABASE_1_ID, TABLE_1_ID, USER_1_PRINCIPAL); @@ -162,8 +162,8 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest { @Test @WithMockUser(username = USER_2_USERNAME, roles = {"DEVELOPER"}) - public void findHistory_developer_succeeds() throws UserNotFoundException, TableNotFoundException, - QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException { + public void findHistory_developer_succeeds()throws TableNotFoundException, QueryStoreException, + QueryMalformedException, DatabaseNotFoundException { /* test */ final List<TableHistoryDto> response = tableService.findHistory(DATABASE_1_ID, TABLE_1_ID, USER_2_PRINCIPAL); @@ -174,8 +174,8 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest { @Test @WithMockUser(username = USER_3_USERNAME, roles = {"DATA_STEWARD"}) - public void findHistory_dataSteward_succeeds() throws UserNotFoundException, TableNotFoundException, - QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException { + public void findHistory_dataSteward_succeeds() throws TableNotFoundException, QueryStoreException, + QueryMalformedException, DatabaseNotFoundException { /* test */ final List<TableHistoryDto> response = tableService.findHistory(DATABASE_1_ID, TABLE_1_ID, USER_3_PRINCIPAL); diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationWriteTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationWriteTest.java index c174c3e3b7fda1be798b8629b7363b6f394a82d6..685604fa7863db9ff3fef2269aaf339824c81ae2 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationWriteTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationWriteTest.java @@ -6,16 +6,10 @@ import at.tuwien.annotations.MockListeners; import at.tuwien.annotations.MockOpensearch; import at.tuwien.api.database.table.TableCreateDto; import at.tuwien.api.database.table.columns.ColumnCreateDto; -import at.tuwien.api.database.table.columns.ColumnTypeDto; -import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto; -import at.tuwien.api.database.table.constraints.ConstraintsCreateDto; import at.tuwien.config.MariaDbConfig; import at.tuwien.config.MariaDbContainerConfig; import at.tuwien.entities.database.table.Table; import at.tuwien.entities.database.table.columns.TableColumn; -import at.tuwien.entities.database.table.columns.TableColumnConcept; -import at.tuwien.entities.identifier.Identifier; -import at.tuwien.entities.identifier.IdentifierType; import at.tuwien.exception.*; import at.tuwien.mapper.TableMapper; import at.tuwien.repository.mdb.*; @@ -41,7 +35,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.any; @Log4j2 @Testcontainers @@ -66,9 +59,6 @@ public class TableServiceIntegrationWriteTest extends BaseUnitTest { @Autowired private LicenseRepository licenseRepository; - @Autowired - private IdentifierRepository identifierRepository; - @Autowired private TableService tableService; @@ -124,9 +114,9 @@ public class TableServiceIntegrationWriteTest extends BaseUnitTest { } @Test - public void create_withConstraints_succeeds() throws UserNotFoundException, TableMalformedException, - QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, TableNameExistsException, - ContainerNotFoundException, SQLException, TableNotFoundException { + public void create_withConstraints_succeeds() throws TableMalformedException, QueryMalformedException, + DatabaseNotFoundException, ImageNotSupportedException, TableNameExistsException, SQLException, + TableNotFoundException { /* test */ tableService.createTable(DATABASE_1_ID, TABLE_5_CREATE_DTO, USER_1_PRINCIPAL); // table to reference @@ -178,7 +168,7 @@ public class TableServiceIntegrationWriteTest extends BaseUnitTest { if (columnEntity.getInternalName().equals("id")) { continue; } - log.trace("internalName={}, type={}", columnEntity.getInternalName(), columnEntity.getColumnType()); + log.trace("internalName={}, type={}, size={}", columnEntity.getInternalName(), columnEntity.getColumnType(), columnEntity.getSize()); /* correct in the metadata database */ assertEquals(columnRequest.getNullAllowed(), columnEntity.getIsNullAllowed()); assertEquals(columnRequest.getPrimaryKey(), columnEntity.getIsPrimaryKey()); @@ -201,23 +191,6 @@ public class TableServiceIntegrationWriteTest extends BaseUnitTest { }); } - @Test - @Transactional - public void update_succeeds() throws TableNotFoundException, SemanticEntityPersistException, - TableMalformedException, QueryMalformedException, DatabaseNotFoundException, - SemanticEntityNotFoundException, ContainerNotFoundException { - final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder() - .conceptUri(COLUMN_CONCEPT_PRECIPITATION_URI) - .build(); - - /* test */ - final TableColumn response = tableService.update(DATABASE_1_ID, TABLE_1_ID, TABLE_1_COLUMNS.get(0).getId(), - request, "abc"); - assertNotNull(response.getConcept()); - final TableColumnConcept concept = response.getConcept(); - assertEquals(COLUMN_CONCEPT_PRECIPITATION_URI, concept.getUri()); - } - @Test @Transactional public void delete_succeeds() throws TableNotFoundException, TableMalformedException, QueryMalformedException, @@ -230,8 +203,7 @@ public class TableServiceIntegrationWriteTest extends BaseUnitTest { @Test @Transactional public void delete_full_succeeds() throws TableNotFoundException, TableMalformedException, QueryMalformedException, - DatabaseNotFoundException, ImageNotSupportedException, UserNotFoundException, TableNameExistsException, - ContainerNotFoundException { + DatabaseNotFoundException, ImageNotSupportedException, TableNameExistsException { /* test */ final Table response = tableService.createTable(DATABASE_1_ID, TABLE_0_CREATE_DTO, USER_1_PRINCIPAL); diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java index c1b225310ad8bdcac8495eb6fe9fd8fe9c138bd6..8ce238c0a02eb4b9052bb567db6babc01a64869d 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java @@ -7,6 +7,7 @@ import at.tuwien.api.database.ViewCreateDto; import at.tuwien.config.MariaDbConfig; import at.tuwien.config.MariaDbContainerConfig; import at.tuwien.entities.database.View; +import at.tuwien.entities.database.ViewColumn; import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.exception.*; import at.tuwien.repository.mdb.*; @@ -14,6 +15,7 @@ import lombok.extern.log4j.Log4j2; import org.junit.Rule; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.rules.Timeout; @@ -91,7 +93,8 @@ public class ViewServiceIntegrationTest extends BaseUnitTest { } @BeforeEach - public void beforeEach() { + public void beforeEach() throws DatabaseUnchangedException, QueryMalformedException, ColumnParseException, + DatabaseNotFoundException, TableMalformedException { TABLE_1.setColumns(TABLE_1_COLUMNS); TABLE_2.setColumns(TABLE_2_COLUMNS); TABLE_3.setColumns(TABLE_3_COLUMNS); @@ -101,6 +104,10 @@ public class ViewServiceIntegrationTest extends BaseUnitTest { TABLE_7.setColumns(TABLE_7_COLUMNS); DATABASE_1.setAccesses(List.of()); DATABASE_2.setAccesses(List.of()); + VIEW_1.setColumns(VIEW_1_COLUMNS); + VIEW_2.setColumns(VIEW_2_COLUMNS); + VIEW_3.setColumns(VIEW_3_COLUMNS); + VIEW_4.setColumns(VIEW_4_COLUMNS); /* metadata database */ imageRepository.save(IMAGE_1); licenseRepository.save(LICENSE_1); @@ -170,34 +177,4 @@ public class ViewServiceIntegrationTest extends BaseUnitTest { assertNull(row2.get("lng")); } - @Test - public void create_withAlias_succeeds() throws DatabaseNotFoundException, UserNotFoundException, - DatabaseConnectionException, ViewMalformedException, QueryMalformedException { - final ViewCreateDto request = ViewCreateDto.builder() - .name(VIEW_2_NAME + "_with_alias") - .query(VIEW_2_QUERY) - .isPublic(VIEW_2_PUBLIC) - .build(); - - /* test */ - final View response = viewService.create(DATABASE_1_ID, request, USER_1_PRINCIPAL); - assertEquals(VIEW_2_NAME + "_with_alias", response.getName()); - assertEquals(VIEW_2_INTERNAL_NAME + "_with_alias", response.getInternalName()); - assertEquals(VIEW_2_QUERY, response.getQuery()); - final List<TableColumn> columns = response.getColumns(); - assertEquals(4, columns.size()); - final TableColumn column0 = columns.get(0); - assertEquals("date", column0.getInternalName()); - assertNull(column0.getAlias()); - final TableColumn column1 = columns.get(1); - assertEquals("location", column1.getInternalName()); - assertEquals("loc", column1.getAlias()); - final TableColumn column2 = columns.get(2); - assertEquals("rainfall", column2.getInternalName()); - assertNull(column2.getAlias()); - final TableColumn column3 = columns.get(3); - assertEquals("mintemp", column3.getInternalName()); - assertNull(column3.getAlias()); - } - } diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServicePersistenceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServicePersistenceIntegrationTest.java index 3480c1682a7cd304717f1c74db3864b788905b39..d57aad698548fd4687ab99b64f4adca2a4e7dc4b 100644 --- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServicePersistenceIntegrationTest.java +++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServicePersistenceIntegrationTest.java @@ -121,7 +121,7 @@ public class ViewServicePersistenceIntegrationTest extends BaseUnitTest { assertEquals(VIEW_1_NAME, response.getName()); assertEquals(VIEW_1_INTERNAL_NAME, response.getInternalName()); assertEquals(VIEW_1_QUERY, response.getQuery()); - assertEquals(VIEW_1_COLUMNS.size(), response.getColumns().size()); + } } diff --git a/dbrepo-metadata-service/rest-service/src/test/resources/csv/weather_aus_lastlinenull.csv b/dbrepo-metadata-service/rest-service/src/test/resources/csv/weather_aus_lastlinenull.csv new file mode 100644 index 0000000000000000000000000000000000000000..12353bbaf7dc5468b5b6e4e904642a0c209ae21f --- /dev/null +++ b/dbrepo-metadata-service/rest-service/src/test/resources/csv/weather_aus_lastlinenull.csv @@ -0,0 +1 @@ +4,2024-01-27,Vienna,, \ No newline at end of file diff --git a/dbrepo-metadata-service/rest-service/src/test/resources/init/weather.sql b/dbrepo-metadata-service/rest-service/src/test/resources/init/weather.sql index 76f78ed34a0e64da912440d18ea61f3fef2595c7..b6dd8cff4cd1dabf0848697cb8db26db4e798601 100644 --- a/dbrepo-metadata-service/rest-service/src/test/resources/init/weather.sql +++ b/dbrepo-metadata-service/rest-service/src/test/resources/init/weather.sql @@ -118,7 +118,7 @@ VALUES ('2022-12-24 17:00:00', 10.0), CREATE VIEW junit2 AS ( -select `date`, `location`, `mintemp`, `rainfall` +select `date`, `location` as loc, `location`, `mintemp`, `rainfall` from `weather_aus` where `location` = 'Albury'); diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/listener/impl/MariadbListenerImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/listener/impl/MariadbListenerImpl.java index 955cd10edd1b60cea7d429a56433aed1d505d376..d227a228bc9e544904a4c1bf238a0dbc219a583e 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/listener/impl/MariadbListenerImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/listener/impl/MariadbListenerImpl.java @@ -40,12 +40,13 @@ public class MariadbListenerImpl implements DatabaseListener { @Override @Scheduled(fixedRateString = "${fda.obtainMetadataRate}", timeUnit = TimeUnit.SECONDS) @Transactional - public void updateStoredMetadata() throws QueryMalformedException, ColumnParseException, - DatabaseNotFoundException, TableNotFoundException { + public void updateStoredMetadata() throws QueryMalformedException, ColumnParseException, DatabaseNotFoundException { for (Long databaseId : databaseRepository.findAllOnlyIds()) { try { - databaseService.obtainMetadata(databaseId); - } catch (DatabaseUnchangedException e) { + databaseService.obtainTablesMetadata(databaseId); + databaseService.obtainConstraints(databaseId); + databaseService.obtainViewsMetadata(databaseId); + } catch (DatabaseUnchangedException | TableMalformedException e) { /* ignore */ } } diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java index 59c0743a2e37a094f0107874755b3c78013163ba..e8045355d30e176c7aff17ce5f5cbae8bce51733 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java @@ -114,7 +114,31 @@ public interface DatabaseService { Database modifyImage(Long databaseId, byte[] image) throws DatabaseNotFoundException; /** - * Obtain metadata from database with given id to read table and view information (schema) and write it to the metadata database for management by DBRepo. + * Obtain table schema constraints for a database by given id. + * + * @param databaseId The database id. + * @return The updated database. + * @throws DatabaseNotFoundException The database was not found in the metadata database. + * @throws QueryMalformedException The inspect query (table/view) is malformed and has syntax issues. + * @throws TableMalformedException The table constraints are malformed. + */ + Database obtainConstraints(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException, TableMalformedException; + + /** + * Obtain metadata from database with given id to read table information (schema) and write it to the metadata database for management by DBRepo. + * + * @param databaseId The database id. + * @return The updated database. + * @throws DatabaseNotFoundException The database was not found in the metadata database. + * @throws QueryMalformedException The inspect query (table/view) is malformed and has syntax issues. + * @throws DatabaseUnchangedException The metadata database is up-to-date and knows about all tables/views in the data database(s). + * @throws ColumnParseException The columns could not be automatically parsed from the views. + */ + Database obtainTablesMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException, + DatabaseUnchangedException, ColumnParseException; + + /** + * Obtain metadata from database with given id to read view information (schema) and write it to the metadata database for management by DBRepo. * * @param databaseId The database id. * @return The updated database. @@ -123,6 +147,6 @@ public interface DatabaseService { * @throws DatabaseUnchangedException The metadata database is up-to-date and knows about all tables/views in the data database(s). * @throws ColumnParseException The columns could not be automatically parsed from the views. */ - Database obtainMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException, - DatabaseUnchangedException, ColumnParseException, TableNotFoundException; + Database obtainViewsMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException, + DatabaseUnchangedException, ColumnParseException; } diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java index d6350915820967217193eeceffae6a359a16484b..4d1f4679f71aa9212dd4bfadb9cf7e1a5597fef5 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java @@ -89,6 +89,7 @@ public interface QueryService { /** * Select all data known in the database-table id tuple at a given time and return a page of specific size, using * Instant to better abstract time concept (JDK 8) from SQL. We use the "mariadb" user for this. + * Precondition: page and size is not null * * @param databaseId The database id. * @param tableId The table id. diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableColumnService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableColumnService.java new file mode 100644 index 0000000000000000000000000000000000000000..5b486d986923e3cbe752d42bbee3f8326640c360 --- /dev/null +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableColumnService.java @@ -0,0 +1,60 @@ +package at.tuwien.service; + +import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto; +import at.tuwien.entities.database.Database; +import at.tuwien.entities.database.table.Table; +import at.tuwien.entities.database.table.columns.TableColumn; +import at.tuwien.exception.DatabaseNotFoundException; +import at.tuwien.exception.TableMalformedException; +import at.tuwien.exception.TableNotFoundException; +import org.springframework.transaction.annotation.Transactional; + +public interface TableColumnService { + + /** + * Updates a table column + * + * @param databaseId The database id. + * @param tableId The table id. + * @param columnId The column id. + * @param updateDto The update data containing unit and concept uris. + * @return The updated table column, if successful. + * @throws TableNotFoundException The table was not found in the metadata database. + * @throws DatabaseNotFoundException The database was not found in the metadata database. + * @throws TableMalformedException The table seems malformed by the mapper. + * @throws TableNotFoundException The table is not found. + */ + TableColumn update(Long databaseId, Long tableId, Long columnId, ColumnSemanticsUpdateDto updateDto) + throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException; + + /** + * Finds a column in a given table with column id + * + * @param table The table. + * @param columnId The column id. + * @return The column, if successful. + * @throws TableMalformedException The requested column was not found in the table. + */ + TableColumn findColumn(Table table, Long columnId) throws TableMalformedException; + + /** + * Finds a column in a given table with column name. + * + * @param table The table. + * @param name The column name. + * @return The column, if successful. + * @throws TableMalformedException The requested column was not found in the table. + */ + TableColumn findColumn(Table table, String name) throws TableMalformedException; + + /** + * Finds a column in a database with given table name and given column name. + * + * @param database The database. + * @param tableName The table name. + * @param columnName The column name. + * @return The column, if successful. + * @throws TableMalformedException The requested column was not found in the database. + */ + TableColumn findColumn(Database database, String tableName, String columnName) throws TableMalformedException; +} diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableService.java index e1e57273a72923e0455635dd4cb53b525dcff08e..09f8012670437c33cc8ab0180a9cb74ea1361259 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableService.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableService.java @@ -84,24 +84,6 @@ public interface TableService { throws ImageNotSupportedException, DatabaseNotFoundException, TableMalformedException, TableNameExistsException, QueryMalformedException, TableNotFoundException; - - /** - * Updates a table column - * - * @param databaseId The database id. - * @param tableId The table id. - * @param columnId The column id. - * @param updateDto The update data containing unit and concept uris. - * @return The updated table column, if successful. - * @throws TableNotFoundException The table was not found in the metadata database. - * @throws DatabaseNotFoundException The database was not found in the metadata database. - * @throws TableMalformedException The table seems malformed by the mapper. - * @throws TableNotFoundException The table is not found. - */ - TableColumn update(Long databaseId, Long tableId, Long columnId, ColumnSemanticsUpdateDto updateDto, - String authorization) throws TableNotFoundException, DatabaseNotFoundException, - TableMalformedException; - /** * Deletes a table from the database in the metadata database and data database. * diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java index 1b675b0fac3e2ebecf7eb8349ced51f8e8f3d15a..183f05711653ca47179a212a2797b05bc1059e38 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java @@ -1,7 +1,6 @@ package at.tuwien.service.impl; import at.tuwien.api.database.DatabaseCreateDto; -import at.tuwien.api.database.DatabaseModifyImageDto; import at.tuwien.api.database.DatabaseModifyVisibilityDto; import at.tuwien.api.database.DatabaseTransferDto; import at.tuwien.config.QueryConfig; @@ -10,20 +9,22 @@ import at.tuwien.entities.container.image.ContainerImageDate; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.View; import at.tuwien.entities.database.table.Table; +import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.entities.database.table.constraints.Constraints; import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKey; +import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKeyReference; +import at.tuwien.entities.database.table.constraints.foreignKey.ReferenceType; import at.tuwien.entities.database.table.constraints.unique.Unique; import at.tuwien.entities.user.User; import at.tuwien.exception.*; import at.tuwien.mapper.DatabaseMapper; import at.tuwien.mapper.QueryMapper; import at.tuwien.mapper.TableMapper; +import at.tuwien.mapper.ViewMapper; import at.tuwien.repository.mdb.ContainerRepository; import at.tuwien.repository.mdb.DatabaseRepository; import at.tuwien.repository.sdb.DatabaseIdxRepository; -import at.tuwien.service.ContainerService; -import at.tuwien.service.DatabaseService; -import at.tuwien.service.UserService; +import at.tuwien.service.*; import com.mchange.v2.c3p0.ComboPooledDataSource; import lombok.extern.log4j.Log4j2; import net.sf.jsqlparser.JSQLParserException; @@ -42,6 +43,7 @@ import java.util.*; @Service public class MariaDbServiceImpl extends HibernateConnector implements DatabaseService { + private final ViewMapper viewMapper; private final QueryConfig queryConfig; private final QueryMapper queryMapper; private final TableMapper tableMapper; @@ -49,14 +51,17 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe private final DatabaseMapper databaseMapper; private final ContainerService containerService; private final DatabaseRepository databaseRepository; + private final TableColumnService tableColumnService; private final ContainerRepository containerRepository; private final DatabaseIdxRepository databaseIdxRepository; @Autowired - public MariaDbServiceImpl(QueryConfig queryConfig, QueryMapper queryMapper, TableMapper tableMapper, - UserService userService, DatabaseMapper databaseMapper, ContainerService containerService, - DatabaseRepository databaseRepository, ContainerRepository containerRepository, + public MariaDbServiceImpl(ViewMapper viewMapper, QueryConfig queryConfig, QueryMapper queryMapper, + TableMapper tableMapper, UserService userService, DatabaseMapper databaseMapper, + ContainerService containerService, DatabaseRepository databaseRepository, + TableColumnService tableColumnService, ContainerRepository containerRepository, DatabaseIdxRepository databaseIdxRepository) { + this.viewMapper = viewMapper; this.queryConfig = queryConfig; this.queryMapper = queryMapper; this.tableMapper = tableMapper; @@ -64,6 +69,7 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe this.databaseMapper = databaseMapper; this.containerService = containerService; this.databaseRepository = databaseRepository; + this.tableColumnService = tableColumnService; this.containerRepository = containerRepository; this.databaseIdxRepository = databaseIdxRepository; } @@ -230,13 +236,47 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe @Override @Transactional - public Database obtainMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException, + public Database obtainConstraints(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException, + TableMalformedException { + /* check */ + final Database database = findById(databaseId); + final List<Table> diffTables = database.getTables() + .stream() + .filter(t -> !t.getProcessedConstraints()) + .toList(); + /* obtain constraints */ + log.info("Database with id {} contains {} table(s) with unknown constraint(s)", databaseId, diffTables.size()); + final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(), database.getContainer(), database); + try { + final Connection connection = dataSource.getConnection(); + for (Table table : diffTables) { + final PreparedStatement preparedStatement = queryMapper.databaseToDatabaseConstraintMetadata(connection, table.getDatabase().getInternalName(), table.getInternalName()); + final Constraints constraints = resultSetTableToObtainedConstraintsMetadata(databaseId, table, preparedStatement.executeQuery()); + table.setConstraints(constraints); + table.setProcessedConstraints(true); + } + } catch (SQLException e) { + log.error("Failed to obtain constraint information in database with id {}: {}", database.getId(), e.getMessage()); + throw new QueryMalformedException("Failed to obtain constraint information in database with id " + database.getId() + ": " + e.getMessage(), e); + } finally { + dataSource.close(); + } + /* update in metadata database */ + final Database entity = databaseRepository.save(database); + /* save in open search database */ + databaseIdxRepository.save(databaseMapper.databaseToDatabaseDto(entity)); + log.info("Updated database with id {} in metadata database & search database", entity.getId()); + return entity; + } + + @Override + @Transactional + public Database obtainTablesMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException, ColumnParseException { /* check */ final Database database = findById(databaseId); final List<Table> diffTables; final List<Table> knownTables; - final List<View> diffViews; final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(), database.getContainer(), database); try { final Connection connection = dataSource.getConnection(); @@ -266,12 +306,6 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe return obtainedTable; }) .toList(); - final List<View> views = tableMapper.resultListToViewList(preparedStatement0.executeQuery(), database); - diffViews = views.stream() - .filter(view -> database.getViews() - .stream() - .noneMatch(v -> v.getInternalName().equals(view.getInternalName()))) - .toList(); /* default times */ final Optional<ContainerImageDate> defaultDateFormat = containerRepository.findDefaultDateFormat(); if (defaultDateFormat.isEmpty()) { @@ -284,7 +318,7 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe throw new ColumnParseException("Failed to find default timestamp format in metadata database"); } /* obtain table schema */ - log.info("Database with id {} contains {} unknown table(s) and {} unknown view(s)", databaseId, diffTables.size(), diffViews.size()); + log.info("Database with id {} contains {} unknown table(s)", databaseId, diffTables.size()); log.debug("database with id {} misses table(s) in metadata database: {}", databaseId, diffTables.stream().map(Table::getInternalName).toList()); database.getTables().replaceAll(table -> { final Optional<Table> optional = knownTables.stream() @@ -306,8 +340,7 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe preparedStatement2.execute(); log.info("Enabled system-versioning for table with name {}", table.getInternalName()); } - final PreparedStatement preparedStatement2 = queryMapper.databaseToDatabaseConstraintMetadata(connection, table.getDatabase().getInternalName(), table.getInternalName()); - table.setConstraints(resultSetTableToObtainedConstraintsMetadata(preparedStatement2.executeQuery())); + table.setProcessedConstraints(false); final PreparedStatement preparedStatement3 = tableMapper.tableToCreateHistoryViewRawQuery(connection, table); preparedStatement3.executeUpdate(); database.getTables().add(table); @@ -318,15 +351,63 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe } finally { dataSource.close(); } + /* update in metadata database */ + final Database entity = databaseRepository.save(database); + /* save in open search database */ + databaseIdxRepository.save(databaseMapper.databaseToDatabaseDto(entity)); + log.info("Updated database with id {} in metadata database & search database", entity.getId()); + return entity; + } + + @Override + @Transactional + public Database obtainViewsMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException, + ColumnParseException { + /* check */ + final Database database = findById(databaseId); + final List<View> diffViews; + final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(), database.getContainer(), database); + try { + final Connection connection = dataSource.getConnection(); + final PreparedStatement preparedStatement0 = databaseMapper.databaseToDatabaseMetadata(connection, database); + final List<View> views = tableMapper.resultListToViewList(preparedStatement0.executeQuery(), database); + diffViews = views.stream() + .filter(view -> database.getViews() + .stream() + .noneMatch(v -> v.getInternalName().equals(view.getInternalName()))) + .toList(); + /* obtain table schema */ + log.info("Database with id {} contains {} unknown view(s)", databaseId, diffViews.size()); + /* default times */ + final Optional<ContainerImageDate> defaultDateFormat = containerRepository.findDefaultDateFormat(); + if (defaultDateFormat.isEmpty()) { + log.error("Failed to find default date format in metadata database"); + throw new ColumnParseException("Failed to find default date format in metadata database"); + } + final Optional<ContainerImageDate> defaultTimestampFormat = containerRepository.findDefaultTimestampFormat(); + if (defaultTimestampFormat.isEmpty()) { + log.error("Failed to find default timestamp format in metadata database"); + throw new ColumnParseException("Failed to find default timestamp format in metadata database"); + } + } catch (SQLException e) { + log.error("Failed to obtain schema information in database with id {}: {}", database.getId(), e.getMessage()); + throw new QueryMalformedException("Failed to obtain schema information in database with id " + database.getId() + ": " + e.getMessage(), e); + } finally { + dataSource.close(); + } /* obtain view schema */ log.debug("database with id {} misses view(s) in metadata database: {}", databaseId, diffViews.stream().map(View::getInternalName).toList()); for (View view : diffViews) { try { - view.setColumns(queryMapper.parseColumns(view.getQuery(), database)); + view.setColumns(viewMapper.tableColumnsToViewColumns(view, queryMapper.parseColumns(view.getQuery(), database))); } catch (JSQLParserException e) { log.error("Failed to map/parse columns: {}", e.getMessage()); throw new ColumnParseException("Failed to map/parse columns: " + e.getMessage(), e); } + if (view.getColumns().stream().anyMatch(c -> c.getColumn().getId() == null)) { + log.warn("Skipping creation of view {}: referenced columns does not exist in metadata database", view.getInternalName()); + continue; + } database.getViews() .add(view); } @@ -338,16 +419,75 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe return entity; } - @Transactional - public Constraints resultSetTableToObtainedConstraintsMetadata(ResultSet resultSet) throws SQLException { + @Transactional(readOnly = true) + public Constraints resultSetTableToObtainedConstraintsMetadata(Long databaseId, Table table, ResultSet resultSet) + throws SQLException, DatabaseNotFoundException, TableMalformedException { + final Database database = find(databaseId); final Set<String> checks = new LinkedHashSet<>(); final List<Unique> uniques = new LinkedList<>(); final List<ForeignKey> foreignKeys = new LinkedList<>(); while (resultSet.next()) { if (resultSet.getString(1).equals("CHECK")) { + /* check constraints */ checks.add(resultSet.getString(4)); } else if (resultSet.getString(1).equals("FOREIGN KEY")) { - // TODO + /* foreign key constraints */ + final List<ForeignKeyReference> foreignKeyReferences = new LinkedList<>(); + final String foreignKeyName = resultSet.getString(2); + if (foreignKeys.stream().anyMatch(fk -> fk.getName().equals(foreignKeyName))) { + final Optional<ForeignKey> optional = foreignKeys.stream() + .filter(fk -> fk.getName().equals(foreignKeyName)) + .findFirst(); + if (optional.isEmpty()) { + /* should never happen */ + continue; + } + final ForeignKey foreignKey = optional.get(); + foreignKey.getReferences() + .add(queryMapper.foreignKeyToForeignKeyReference(foreignKey, + tableColumnService.findColumn(database, resultSet.getString(6), resultSet.getString(8)), + tableColumnService.findColumn(table, resultSet.getString(7)))); + } + final ForeignKey foreignKey; + try { + foreignKey = ForeignKey.builder() + .name(foreignKeyName) + .table(table) + .referencedTable(find(database, resultSet.getString(6))) + .references(foreignKeyReferences) + .onDelete(ReferenceType.NO_ACTION) + .onUpdate(ReferenceType.NO_ACTION) + .build(); + } catch (TableNotFoundException e) { + /* ignore */ + return null; + } + final ForeignKeyReference fk = ForeignKeyReference.builder() + .foreignKey(foreignKey) + .column(tableColumnService.findColumn(table, resultSet.getString(7))) + .referencedColumn(tableColumnService.findColumn(database, resultSet.getString(6), resultSet.getString(8))) + .build(); + foreignKey.setReferences(List.of(fk)); + foreignKeys.add(foreignKey); + } else if (resultSet.getString(1).equals("UNIQUE")) { + /* unique constraints */ + final String uniqueConstraintName = resultSet.getString(1); + final Optional<Unique> optional = uniques.stream().filter(u -> u.getName().equals(uniqueConstraintName)).findFirst(); + if (optional.isPresent()) { + log.debug("unique constraint {} already present: add column", uniqueConstraintName); + optional.get() + .getColumns() + .add(tableColumnService.findColumn(table, resultSet.getString(7))); + continue; + } + final List<TableColumn> columns = new LinkedList<>(); + columns.add(tableColumnService.findColumn(table, resultSet.getString(7))); + final Unique uk = Unique.builder() + .name(uniqueConstraintName) + .table(table) + .columns(columns) + .build(); + uniques.add(uk); } } final Constraints constraints = Constraints.builder() @@ -361,4 +501,16 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe return constraints; } + public Table find(Database database, String internalName) throws DatabaseNotFoundException, TableNotFoundException { + final Optional<Table> table = database.getTables() + .stream() + .filter(t -> t.getInternalName().equals(internalName)) + .findFirst(); + if (table.isEmpty()) { + log.error("Failed to find table with internal name {} in metadata database", internalName); + throw new TableNotFoundException("Failed to find table with internal name " + internalName + " in metadata database"); + } + return table.get(); + } + } diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java index 775df9e82dda9c5604114faeebd840b37c9284cb..4b1d5bb95291a4a825e7513d88fbdf23794315e8 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java @@ -27,10 +27,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.security.Principal; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; +import java.sql.*; import java.time.Instant; import java.time.format.DateTimeParseException; import java.util.List; @@ -120,7 +117,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService return executeCountNonPersistent(databaseId, statement); } - private PreparedStatement prepareStatement(Connection connection, String statement) throws QueryMalformedException { + public PreparedStatement prepareStatement(Connection connection, String statement) throws QueryMalformedException { try { return connection.prepareStatement(statement); } catch (SQLException e) { @@ -129,7 +126,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService } } - private QueryResultDto executeNonPersistent(Long databaseId, String statement, List<TableColumn> columns) + public QueryResultDto executeNonPersistent(Long databaseId, String statement, List<TableColumn> columns) throws QueryMalformedException, DatabaseNotFoundException, TableMalformedException { /* find */ final Database database = databaseService.find(databaseId); @@ -150,7 +147,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService } } - private Long executeCountNonPersistent(Long databaseId, String statement) + public Long executeCountNonPersistent(Long databaseId, String statement) throws QueryMalformedException, TableMalformedException, DatabaseNotFoundException, QueryStoreException { /* find */ final Database database = databaseService.find(databaseId); @@ -195,7 +192,11 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService final Connection connection = dataSource.getConnection(); final PreparedStatement preparedStatement = viewMapper.viewToSelectAll(connection, view, page, size); final ResultSet resultSet = preparedStatement.executeQuery(); - return queryMapper.resultListToQueryResultDto(view.getColumns(), resultSet); + final List<TableColumn> columns = view.getColumns() + .stream() + .map(viewMapper::viewColumnToTableColumn) + .toList(); + return queryMapper.resultListToQueryResultDto(columns, resultSet); } catch (SQLException e) { log.error("Failed to map object: {}", e.getMessage()); throw new TableMalformedException("Failed to map object: " + e.getMessage(), e); @@ -249,7 +250,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService return retrieveBlobAsResource(database.getContainer(), filename); } - private ExportResource retrieveBlobAsResource(Container container, String filename) throws DataDbSidecarException, + public ExportResource retrieveBlobAsResource(Container container, String filename) throws DataDbSidecarException, FileStorageException, DataProcessingException { /* upload from sidecar into blob storage */ dataDbSidecarGateway.exportFile(container.getSidecarHost(), container.getSidecarPort(), filename); @@ -327,7 +328,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService final Database database = databaseService.find(databaseId); final Table table = tableService.find(databaseId, tableId); /* run query */ - if (data.getKeys().size() == 0) return; + if (data.getKeys().isEmpty()) return; final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(), database.getContainer(), database); /* prepare the statement */ @@ -358,8 +359,12 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService database.getContainer(), database); try { final Connection connection = dataSource.getConnection(); - queryMapper.importCsvQuery(connection, table, data); + final PreparedStatement statement = queryMapper.pathToRawInsertQuery(connection, table, data); + statement.executeUpdate(); } catch (SQLException e) { + log.error("Failed to open connection to data database: {}", e.getMessage()); + throw new TableMalformedException("Failed to open connection to data database: " + e.getMessage(), e); + } catch (QueryMalformedException e) { log.error("Failed to import csv: {}", e.getMessage()); throw new TableMalformedException("Failed to import csv: " + e.getMessage(), e); } finally { diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableColumnServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableColumnServiceImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..46763db156bd332ed2e3385cc473230c94dd0ccf --- /dev/null +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableColumnServiceImpl.java @@ -0,0 +1,153 @@ +package at.tuwien.service.impl; + +import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto; +import at.tuwien.entities.database.Database; +import at.tuwien.entities.database.table.Table; +import at.tuwien.entities.database.table.columns.TableColumn; +import at.tuwien.entities.database.table.columns.TableColumnConcept; +import at.tuwien.entities.database.table.columns.TableColumnUnit; +import at.tuwien.exception.*; +import at.tuwien.mapper.DatabaseMapper; +import at.tuwien.repository.mdb.DatabaseRepository; +import at.tuwien.repository.sdb.DatabaseIdxRepository; +import at.tuwien.service.SemanticService; +import at.tuwien.service.TableColumnService; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +@Log4j2 +@Service +public class TableColumnServiceImpl implements TableColumnService { + + private final DatabaseMapper databaseMapper; + private final SemanticService semanticService; + private final DatabaseRepository databaseRepository; + private final DatabaseIdxRepository databaseIdxRepository; + + @Autowired + public TableColumnServiceImpl(DatabaseMapper databaseMapper, SemanticService semanticService, + DatabaseRepository databaseRepository, DatabaseIdxRepository databaseIdxRepository) { + this.databaseMapper = databaseMapper; + this.semanticService = semanticService; + this.databaseRepository = databaseRepository; + this.databaseIdxRepository = databaseIdxRepository; + } + + @Transactional(readOnly = true) + public Database find(Long databaseId) throws DatabaseNotFoundException { + final Optional<Database> database = databaseRepository.findById(databaseId); + if (database.isEmpty()) { + log.error("Failed to find database with id {} in metadata database", databaseId); + throw new DatabaseNotFoundException("could not find database with id " + databaseId + " in metadata database"); + } + return database.get(); + } + + @Transactional(readOnly = true) + public Table find(Long databaseId, Long tableId) throws DatabaseNotFoundException, TableNotFoundException { + final Optional<Table> table = find(databaseId) + .getTables() + .stream() + .filter(t -> t.getId().equals(tableId)) + .findFirst(); + if (table.isEmpty()) { + log.error("Failed to find table with id {} in metadata database", tableId); + throw new TableNotFoundException("Failed to find table with id " + tableId + " in metadata database"); + } + return table.get(); + } + + @Override + @Transactional + public TableColumn update(Long databaseId, Long tableId, Long columnId, ColumnSemanticsUpdateDto updateDto) + throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException { + final Table table = find(databaseId, tableId); + final TableColumn column = findColumn(table, columnId); + /* assign */ + if (updateDto.getUnitUri() != null) { + try { + column.setUnit(semanticService.findUnit(updateDto.getUnitUri())); + log.debug("found unit with uri {} in metadata database", updateDto.getUnitUri()); + } catch (UnitNotFoundException e) { + final TableColumnUnit unit = TableColumnUnit.builder() + .uri(updateDto.getUnitUri()) + .build(); + column.setUnit(unit); + } + } else { + column.setUnit(null); + } + if (updateDto.getConceptUri() != null) { + try { + column.setConcept(semanticService.findConcept(updateDto.getConceptUri())); + log.debug("found concept with uri {} in metadata database", updateDto.getConceptUri()); + } catch (ConceptNotFoundException e) { + final TableColumnConcept concept = TableColumnConcept.builder() + .uri(updateDto.getConceptUri()) + .build(); + column.setConcept(concept); + } + } else { + column.setConcept(null); + } + /* update in metadata database */ + table.getColumns().set(table.getColumns().indexOf(column), column); + databaseRepository.save(table.getDatabase()); + /* update in open search database */ + databaseIdxRepository.save(databaseMapper.databaseToDatabaseDto(find(databaseId))); + log.info("Updated table column with id {} of table with id {} in metadata database & search database", columnId, tableId); + return column; + } + + @Override + @Transactional(readOnly = true) + public TableColumn findColumn(Table table, Long columnId) throws TableMalformedException { + final Optional<TableColumn> optional = table.getColumns() + .stream() + .filter(c -> c.getId().equals(columnId)) + .findFirst(); + if (optional.isEmpty()) { + log.error("Failed to find column with id {} in metadata database", columnId); + throw new TableMalformedException("Failed to find column with id " + columnId + " in metadata database"); + } + return optional.get(); + } + + @Override + @Transactional(readOnly = true) + public TableColumn findColumn(Table table, String name) throws TableMalformedException { + final Optional<TableColumn> optional = table.getColumns() + .stream() + .filter(c -> c.getInternalName().equals(name)) + .findFirst(); + if (optional.isEmpty()) { + log.error("Failed to find column with name {} in table with name {}", name, table.getInternalName()); + throw new TableMalformedException("Failed to find column with name " + name + " in table with name " + table.getInternalName()); + } + return optional.get(); + } + + @Override + @Transactional(readOnly = true) + public TableColumn findColumn(Database database, String tableName, String columnName) + throws TableMalformedException { + final Optional<TableColumn> optional = database.getTables() + .stream() + .filter(t -> t.getInternalName().equals(tableName)) + .map(Table::getColumns) + .flatMap(List::stream) + .filter(c -> c.getInternalName().equals(columnName)) + .findFirst(); + if (optional.isEmpty()) { + log.error("Failed to find column {}.{} in database with id {}", tableName, columnName, database.getId()); + throw new TableMalformedException("Failed to find column " + tableName + "." + columnName + " in database with id " + database.getId()); + } + return optional.get(); + } +} diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java index 3201ad9ea0e7b8ea839da97ed6a0b67352309e4d..525383b115cdfc36806dac9677adc076715d955c 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java @@ -40,19 +40,17 @@ public class TableServiceImpl extends HibernateConnector implements TableService private final TableMapper tableMapper; private final DatabaseMapper databaseMapper; private final DatabaseService databaseService; - private final SemanticService semanticService; private final DatabaseRepository databaseRepository; private final DatabaseIdxRepository databaseIdxRepository; @Autowired public TableServiceImpl(QueryMapper queryMapper, TableMapper tableMapper, DatabaseMapper databaseMapper, - DatabaseService databaseService, SemanticService semanticService, - DatabaseRepository databaseRepository, DatabaseIdxRepository databaseIdxRepository) { + DatabaseService databaseService, DatabaseRepository databaseRepository, + DatabaseIdxRepository databaseIdxRepository) { this.queryMapper = queryMapper; this.tableMapper = tableMapper; this.databaseMapper = databaseMapper; this.databaseService = databaseService; - this.semanticService = semanticService; this.databaseRepository = databaseRepository; this.databaseIdxRepository = databaseIdxRepository; } @@ -207,49 +205,6 @@ public class TableServiceImpl extends HibernateConnector implements TableService return optionalEntity.get(); } - @Override - @Transactional - public TableColumn update(Long databaseId, Long tableId, Long columnId, ColumnSemanticsUpdateDto updateDto, - String authorization) throws TableNotFoundException, DatabaseNotFoundException, - TableMalformedException { - final Table table = find(databaseId, tableId); - final TableColumn column = findColumn(table, columnId); - /* assign */ - if (updateDto.getUnitUri() != null) { - try { - column.setUnit(semanticService.findUnit(updateDto.getUnitUri())); - log.debug("found unit with uri {} in metadata database", updateDto.getUnitUri()); - } catch (UnitNotFoundException e) { - final TableColumnUnit unit = TableColumnUnit.builder() - .uri(updateDto.getUnitUri()) - .build(); - column.setUnit(unit); - } - } else { - column.setUnit(null); - } - if (updateDto.getConceptUri() != null) { - try { - column.setConcept(semanticService.findConcept(updateDto.getConceptUri())); - log.debug("found concept with uri {} in metadata database", updateDto.getConceptUri()); - } catch (ConceptNotFoundException e) { - final TableColumnConcept concept = TableColumnConcept.builder() - .uri(updateDto.getConceptUri()) - .build(); - column.setConcept(concept); - } - } else { - column.setConcept(null); - } - /* update in metadata database */ - table.getColumns().set(table.getColumns().indexOf(column), column); - databaseRepository.save(table.getDatabase()); - /* update in open search database */ - databaseIdxRepository.save(databaseMapper.databaseToDatabaseDto(databaseService.find(databaseId))); - log.info("Updated table column with id {} of table with id {} in metadata database & search database", columnId, tableId); - return column; - } - @Override @Transactional public void deleteTable(Long databaseId, Long tableId) @@ -278,24 +233,4 @@ public class TableServiceImpl extends HibernateConnector implements TableService log.info("Deleted table with id {} in open search database", table.getId()); } - /** - * Finds a column in a given table with column id - * - * @param table The table. - * @param columnId The column id. - * @return The column, if successful. - * @throws TableMalformedException The requested column was not found in the table. - */ - protected TableColumn findColumn(Table table, Long columnId) throws TableMalformedException { - final Optional<TableColumn> optional = table.getColumns() - .stream() - .filter(c -> c.getId().equals(columnId)) - .findFirst(); - if (optional.isEmpty()) { - log.error("Failed to find column with id {} in metadata database", columnId); - throw new TableMalformedException("Failed to find column with id " + columnId + " in metadata database"); - } - return optional.get(); - } - } diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java index 2a57cadf190fa111233efb30c5fa13ba384411cb..d9dc02f88a602cde25013bb3feb0f67b78ddcb19 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java @@ -3,6 +3,7 @@ package at.tuwien.service.impl; import at.tuwien.api.database.ViewCreateDto; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.View; +import at.tuwien.entities.database.ViewColumn; import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.exception.*; import at.tuwien.mapper.DatabaseMapper; @@ -146,7 +147,7 @@ public class ViewServiceImpl extends HibernateConnector implements ViewService { columns = queryMapper.parseColumns(data.getQuery(), database); } catch (JSQLParserException e) { log.error("Failed to map/parse columns: {}", e.getMessage()); - throw new QueryMalformedException(e.getMessage(), e); + throw new QueryMalformedException("Failed to map/parse columns: " + e.getMessage(), e); } try { final Connection connection = dataSource.getConnection(); @@ -171,8 +172,8 @@ public class ViewServiceImpl extends HibernateConnector implements ViewService { .toString()) .isInitialView(false) .isPublic(data.getIsPublic()) - .columns(columns) .build(); + entity.setColumns(viewMapper.tableColumnsToViewColumns(entity, columns)); database.getViews() .add(entity); final Optional<View> optional = databaseRepository.save(database) diff --git a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java index 67a2711a98bc1b93492fdfb25102daf6f9c0037b..8fc32dc4671ca372cb5d397b04d5e86c8dcf2727 100644 --- a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java +++ b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java @@ -1334,6 +1334,7 @@ public abstract class BaseTest { public final static String TABLE_1_NAME = "Weather AUS"; public final static String TABLE_1_INTERNALNAME = "weather_aus"; public final static Boolean TABLE_1_VERSIONED = true; + public final static Boolean TABLE_1_PROCESSED_CONSTRAINTS = true; public final static String TABLE_1_DESCRIPTION = "Weather in the world"; public final static String TABLE_1_QUEUE_NAME = TABLE_1_INTERNALNAME; public final static String TABLE_1_ROUTING_KEY = "dbrepo\\." + DATABASE_1_EXCHANGE + "\\." + TABLE_1_QUEUE_NAME; @@ -1348,6 +1349,7 @@ public abstract class BaseTest { .created(TABLE_1_CREATED) .internalName(TABLE_1_INTERNALNAME) .isVersioned(TABLE_1_VERSIONED) + .processedConstraints(TABLE_1_PROCESSED_CONSTRAINTS) .description(TABLE_1_DESCRIPTION) .name(TABLE_1_NAME) .queueName(TABLE_1_QUEUE_NAME) @@ -1393,6 +1395,7 @@ public abstract class BaseTest { public final static String TABLE_2_NAME = "Weather Location"; public final static String TABLE_2_INTERNALNAME = "weather_location"; public final static Boolean TABLE_2_VERSIONED = true; + public final static Boolean TABLE_2_PROCESSED_CONSTRAINTS = true; public final static String TABLE_2_DESCRIPTION = "Weather location"; public final static String TABLE_2_QUEUE_NAME = TABLE_2_INTERNALNAME; public final static String TABLE_2_ROUTING_KEY = "dbrepo\\." + DATABASE_1_EXCHANGE + "\\." + TABLE_2_QUEUE_NAME; @@ -1406,6 +1409,7 @@ public abstract class BaseTest { .created(TABLE_2_CREATED) .internalName(TABLE_2_INTERNALNAME) .isVersioned(TABLE_2_VERSIONED) + .processedConstraints(TABLE_2_PROCESSED_CONSTRAINTS) .description(TABLE_2_DESCRIPTION) .name(TABLE_2_NAME) .lastModified(TABLE_2_LAST_MODIFIED) @@ -1448,6 +1452,7 @@ public abstract class BaseTest { public final static String TABLE_3_NAME = "Sensor"; public final static String TABLE_3_INTERNALNAME = "sensor"; public final static Boolean TABLE_3_VERSIONED = true; + public final static Boolean TABLE_3_PROCESSED_CONSTRAINTS = true; public final static String TABLE_3_DESCRIPTION = "Some sensor data"; public final static String TABLE_3_QUEUE_NAME = TABLE_3_INTERNALNAME; public final static String TABLE_3_ROUTING_KEY = "dbrepo\\." + DATABASE_1_EXCHANGE + "\\." + TABLE_3_QUEUE_NAME; @@ -1461,6 +1466,7 @@ public abstract class BaseTest { .created(TABLE_3_CREATED) .internalName(TABLE_3_INTERNALNAME) .isVersioned(TABLE_3_VERSIONED) + .processedConstraints(TABLE_3_PROCESSED_CONSTRAINTS) .description(TABLE_3_DESCRIPTION) .name(TABLE_3_NAME) .lastModified(TABLE_3_LAST_MODIFIED) @@ -1529,6 +1535,7 @@ public abstract class BaseTest { public final static String TABLE_5_NAME = "zoo"; public final static String TABLE_5_INTERNALNAME = "zoo"; public final static Boolean TABLE_5_VERSIONED = true; + public final static Boolean TABLE_5_PROCESSED_CONSTRAINTS = true; public final static String TABLE_5_DESCRIPTION = "Some Kaggle dataset"; public final static String TABLE_5_QUEUE_NAME = TABLE_5_INTERNALNAME; public final static String TABLE_5_ROUTING_KEY = "dbrepo\\." + DATABASE_2_EXCHANGE + "\\." + TABLE_5_QUEUE_NAME; @@ -1541,6 +1548,7 @@ public abstract class BaseTest { .created(Instant.now()) .internalName(TABLE_5_INTERNALNAME) .isVersioned(TABLE_5_VERSIONED) + .processedConstraints(TABLE_5_PROCESSED_CONSTRAINTS) .description(TABLE_5_DESCRIPTION) .name(TABLE_5_NAME) .lastModified(TABLE_5_LAST_MODIFIED) @@ -1579,6 +1587,7 @@ public abstract class BaseTest { public final static String TABLE_6_NAME = "names"; public final static String TABLE_6_INTERNALNAME = "names"; public final static Boolean TABLE_6_VERSIONED = true; + public final static Boolean TABLE_6_PROCESSED_CONSTRAINTS = true; public final static String TABLE_6_DESCRIPTION = "Some names dataset"; public final static String TABLE_6_QUEUE_NAME = TABLE_6_INTERNALNAME; public final static String TABLE_6_ROUTING_KEY = "dbrepo\\." + DATABASE_2_EXCHANGE + "\\." + TABLE_6_QUEUE_NAME; @@ -1591,6 +1600,7 @@ public abstract class BaseTest { .created(TABLE_6_CREATED) .internalName(TABLE_6_INTERNALNAME) .isVersioned(TABLE_6_VERSIONED) + .processedConstraints(TABLE_6_PROCESSED_CONSTRAINTS) .description(TABLE_6_DESCRIPTION) .name(TABLE_6_NAME) .lastModified(TABLE_6_LAST_MODIFIED) @@ -1625,6 +1635,7 @@ public abstract class BaseTest { public final static String TABLE_7_NAME = "likes"; public final static String TABLE_7_INTERNAL_NAME = "likes"; public final static Boolean TABLE_7_VERSIONED = true; + public final static Boolean TABLE_7_PROCESSED_CONSTRAINTS = true; public final static String TABLE_7_DESCRIPTION = "Some likes dataset"; public final static String TABLE_7_QUEUE_NAME = TABLE_7_INTERNAL_NAME; public final static String TABLE_7_ROUTING_KEY = "dbrepo\\." + DATABASE_2_EXCHANGE + "\\." + TABLE_7_QUEUE_NAME; @@ -1637,6 +1648,7 @@ public abstract class BaseTest { .created(TABLE_7_CREATED) .internalName(TABLE_7_INTERNAL_NAME) .isVersioned(TABLE_7_VERSIONED) + .processedConstraints(TABLE_7_PROCESSED_CONSTRAINTS) .description(TABLE_7_DESCRIPTION) .name(TABLE_7_NAME) .lastModified(TABLE_7_LAST_MODIFIED) @@ -1671,6 +1683,7 @@ public abstract class BaseTest { public final static String TABLE_4_NAME = "Sensor"; public final static String TABLE_4_INTERNAL_NAME = "sensor"; public final static Boolean TABLE_4_VERSIONED = true; + public final static Boolean TABLE_4_PROCESSED_CONSTRAINTS = true; public final static String TABLE_4_DESCRIPTION = "Hello sensor"; public final static String TABLE_4_QUEUE_NAME = TABLE_4_INTERNAL_NAME; public final static String TABLE_4_ROUTING_KEY = "dbrepo\\." + DATABASE_1_EXCHANGE + "\\." + TABLE_4_QUEUE_NAME; @@ -1688,6 +1701,7 @@ public abstract class BaseTest { .routingKey(TABLE_4_ROUTING_KEY) .columns(List.of() /* TABLE_4_COLUMNS */) .isVersioned(TABLE_4_VERSIONED) + .processedConstraints(TABLE_4_PROCESSED_CONSTRAINTS) .createdBy(USER_1_ID) .ownedBy(USER_1_ID) .owner(USER_1) @@ -1772,6 +1786,7 @@ public abstract class BaseTest { public final static String TABLE_8_NAME = "mfcc"; public final static String TABLE_8_INTERNAL_NAME = "mfcc"; public final static Boolean TABLE_8_VERSIONED = true; + public final static Boolean TABLE_8_PROCESSED_CONSTRAINTS = true; public final static String TABLE_8_DESCRIPTION = "Hello mfcc"; public final static String TABLE_8_QUEUE_NAME = TABLE_8_INTERNAL_NAME; public final static String TABLE_8_ROUTING_KEY = "dbrepo\\." + DATABASE_3_EXCHANGE + "\\." + TABLE_8_QUEUE_NAME; @@ -1784,6 +1799,7 @@ public abstract class BaseTest { .internalName(TABLE_8_INTERNAL_NAME) .description(TABLE_8_DESCRIPTION) .isVersioned(TABLE_8_VERSIONED) + .processedConstraints(TABLE_8_PROCESSED_CONSTRAINTS) .database(null /* DATABASE_1 */) .name(TABLE_8_NAME) .queueName(TABLE_8_QUEUE_NAME) @@ -2039,7 +2055,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_1_TYPE_DTO = ColumnTypeDto.BIGINT; public final static Long COLUMN_4_1_DATE_FORMAT = null; public final static Boolean COLUMN_4_1_NULL = false; - public final static Boolean COLUMN_4_1_UNIQUE = true; public final static Boolean COLUMN_4_1_AUTO_GENERATED = true; public final static String COLUMN_4_1_FOREIGN_KEY = null; public final static String COLUMN_4_1_CHECK = null; @@ -2059,7 +2074,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_2_TYPE_DTO = ColumnTypeDto.VARCHAR; public final static Long COLUMN_4_2_DATE_FORMAT = null; public final static Boolean COLUMN_4_2_NULL = true; - public final static Boolean COLUMN_4_2_UNIQUE = false; public final static Boolean COLUMN_4_2_AUTO_GENERATED = false; public final static String COLUMN_4_2_FOREIGN_KEY = null; public final static String COLUMN_4_2_CHECK = null; @@ -2079,7 +2093,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_3_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_3_DATE_FORMAT = null; public final static Boolean COLUMN_4_3_NULL = true; - public final static Boolean COLUMN_4_3_UNIQUE = false; public final static Boolean COLUMN_4_3_AUTO_GENERATED = false; public final static String COLUMN_4_3_FOREIGN_KEY = null; public final static String COLUMN_4_3_CHECK = null; @@ -2099,7 +2112,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_4_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_4_DATE_FORMAT = null; public final static Boolean COLUMN_4_4_NULL = true; - public final static Boolean COLUMN_4_4_UNIQUE = false; public final static Boolean COLUMN_4_4_AUTO_GENERATED = false; public final static String COLUMN_4_4_FOREIGN_KEY = null; public final static String COLUMN_4_4_CHECK = null; @@ -2119,7 +2131,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_5_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_5_DATE_FORMAT = null; public final static Boolean COLUMN_4_5_NULL = true; - public final static Boolean COLUMN_4_5_UNIQUE = false; public final static Boolean COLUMN_4_5_AUTO_GENERATED = false; public final static String COLUMN_4_5_FOREIGN_KEY = null; public final static String COLUMN_4_5_CHECK = null; @@ -2139,7 +2150,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_6_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_6_DATE_FORMAT = null; public final static Boolean COLUMN_4_6_NULL = true; - public final static Boolean COLUMN_4_6_UNIQUE = false; public final static Boolean COLUMN_4_6_AUTO_GENERATED = false; public final static String COLUMN_4_6_FOREIGN_KEY = null; public final static String COLUMN_4_6_CHECK = null; @@ -2159,7 +2169,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_7_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_7_DATE_FORMAT = null; public final static Boolean COLUMN_4_7_NULL = true; - public final static Boolean COLUMN_4_7_UNIQUE = false; public final static Boolean COLUMN_4_7_AUTO_GENERATED = false; public final static String COLUMN_4_7_FOREIGN_KEY = null; public final static String COLUMN_4_7_CHECK = null; @@ -2179,7 +2188,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_8_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_8_DATE_FORMAT = null; public final static Boolean COLUMN_4_8_NULL = true; - public final static Boolean COLUMN_4_8_UNIQUE = false; public final static Boolean COLUMN_4_8_AUTO_GENERATED = false; public final static String COLUMN_4_8_FOREIGN_KEY = null; public final static String COLUMN_4_8_CHECK = null; @@ -2199,7 +2207,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_9_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_9_DATE_FORMAT = null; public final static Boolean COLUMN_4_9_NULL = true; - public final static Boolean COLUMN_4_9_UNIQUE = false; public final static Boolean COLUMN_4_9_AUTO_GENERATED = false; public final static String COLUMN_4_9_FOREIGN_KEY = null; public final static String COLUMN_4_9_CHECK = null; @@ -2219,7 +2226,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_10_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_10_DATE_FORMAT = null; public final static Boolean COLUMN_4_10_NULL = true; - public final static Boolean COLUMN_4_10_UNIQUE = false; public final static Boolean COLUMN_4_10_AUTO_GENERATED = false; public final static String COLUMN_4_10_FOREIGN_KEY = null; public final static String COLUMN_4_10_CHECK = null; @@ -2239,7 +2245,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_11_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_11_DATE_FORMAT = null; public final static Boolean COLUMN_4_11_NULL = true; - public final static Boolean COLUMN_4_11_UNIQUE = false; public final static Boolean COLUMN_4_11_AUTO_GENERATED = false; public final static String COLUMN_4_11_FOREIGN_KEY = null; public final static String COLUMN_4_11_CHECK = null; @@ -2259,7 +2264,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_12_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_12_DATE_FORMAT = null; public final static Boolean COLUMN_4_12_NULL = true; - public final static Boolean COLUMN_4_12_UNIQUE = false; public final static Boolean COLUMN_4_12_AUTO_GENERATED = false; public final static String COLUMN_4_12_FOREIGN_KEY = null; public final static String COLUMN_4_12_CHECK = null; @@ -2279,7 +2283,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_13_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_13_DATE_FORMAT = null; public final static Boolean COLUMN_4_13_NULL = true; - public final static Boolean COLUMN_4_13_UNIQUE = false; public final static Boolean COLUMN_4_13_AUTO_GENERATED = false; public final static String COLUMN_4_13_FOREIGN_KEY = null; public final static String COLUMN_4_13_CHECK = null; @@ -2299,7 +2302,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_14_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_14_DATE_FORMAT = null; public final static Boolean COLUMN_4_14_NULL = true; - public final static Boolean COLUMN_4_14_UNIQUE = false; public final static Boolean COLUMN_4_14_AUTO_GENERATED = false; public final static String COLUMN_4_14_FOREIGN_KEY = null; public final static String COLUMN_4_14_CHECK = null; @@ -2319,7 +2321,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_15_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_15_DATE_FORMAT = null; public final static Boolean COLUMN_4_15_NULL = true; - public final static Boolean COLUMN_4_15_UNIQUE = false; public final static Boolean COLUMN_4_15_AUTO_GENERATED = false; public final static String COLUMN_4_15_FOREIGN_KEY = null; public final static String COLUMN_4_15_CHECK = null; @@ -2339,7 +2340,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_16_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_16_DATE_FORMAT = null; public final static Boolean COLUMN_4_16_NULL = true; - public final static Boolean COLUMN_4_16_UNIQUE = false; public final static Boolean COLUMN_4_16_AUTO_GENERATED = false; public final static String COLUMN_4_16_FOREIGN_KEY = null; public final static String COLUMN_4_16_CHECK = null; @@ -2359,7 +2359,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_17_TYPE_DTO = ColumnTypeDto.INT; public final static Long COLUMN_4_17_DATE_FORMAT = null; public final static Boolean COLUMN_4_17_NULL = true; - public final static Boolean COLUMN_4_17_UNIQUE = false; public final static Boolean COLUMN_4_17_AUTO_GENERATED = false; public final static String COLUMN_4_17_FOREIGN_KEY = null; public final static String COLUMN_4_17_CHECK = null; @@ -2379,7 +2378,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_18_TYPE_DTO = ColumnTypeDto.DECIMAL; public final static Long COLUMN_4_18_DATE_FORMAT = null; public final static Boolean COLUMN_4_18_NULL = true; - public final static Boolean COLUMN_4_18_UNIQUE = false; public final static Boolean COLUMN_4_18_AUTO_GENERATED = false; public final static String COLUMN_4_18_FOREIGN_KEY = null; public final static String COLUMN_4_18_CHECK = null; @@ -2399,7 +2397,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_19_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_19_DATE_FORMAT = null; public final static Boolean COLUMN_4_19_NULL = true; - public final static Boolean COLUMN_4_19_UNIQUE = false; public final static Boolean COLUMN_4_19_AUTO_GENERATED = false; public final static String COLUMN_4_19_FOREIGN_KEY = null; public final static String COLUMN_4_19_CHECK = null; @@ -2419,7 +2416,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_20_TYPE_DTO = ColumnTypeDto.BOOL; public final static Long COLUMN_4_20_DATE_FORMAT = null; public final static Boolean COLUMN_4_20_NULL = true; - public final static Boolean COLUMN_4_20_UNIQUE = false; public final static Boolean COLUMN_4_20_AUTO_GENERATED = false; public final static String COLUMN_4_20_FOREIGN_KEY = null; public final static String COLUMN_4_20_CHECK = null; @@ -2439,7 +2435,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_4_21_TYPE_DTO = ColumnTypeDto.DECIMAL; public final static Long COLUMN_4_21_DATE_FORMAT = null; public final static Boolean COLUMN_4_21_NULL = true; - public final static Boolean COLUMN_4_21_UNIQUE = false; public final static Boolean COLUMN_4_21_AUTO_GENERATED = false; public final static String COLUMN_4_21_FOREIGN_KEY = null; public final static String COLUMN_4_21_CHECK = null; @@ -2459,7 +2454,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_5_1_TYPE_DTO = ColumnTypeDto.BIGINT; public final static Long COLUMN_5_1_DATE_FORMAT = null; public final static Boolean COLUMN_5_1_NULL = false; - public final static Boolean COLUMN_5_1_UNIQUE = true; public final static Boolean COLUMN_5_1_AUTO_GENERATED = true; public final static String COLUMN_5_1_FOREIGN_KEY = null; public final static String COLUMN_5_1_CHECK = null; @@ -2478,7 +2472,6 @@ public abstract class BaseTest { public final static Long COLUMN_5_2_SIZE = 20L; public final static Long COLUMN_5_2_DATE_FORMAT = null; public final static Boolean COLUMN_5_2_NULL = false; - public final static Boolean COLUMN_5_2_UNIQUE = false; public final static Boolean COLUMN_5_2_AUTO_GENERATED = false; public final static String COLUMN_5_2_FOREIGN_KEY = null; public final static String COLUMN_5_2_CHECK = null; @@ -2497,7 +2490,6 @@ public abstract class BaseTest { public final static Long COLUMN_5_3_SIZE = 40L; public final static Long COLUMN_5_3_DATE_FORMAT = null; public final static Boolean COLUMN_5_3_NULL = false; - public final static Boolean COLUMN_5_3_UNIQUE = false; public final static Boolean COLUMN_5_3_AUTO_GENERATED = false; public final static String COLUMN_5_3_FOREIGN_KEY = null; public final static String COLUMN_5_3_CHECK = null; @@ -2515,7 +2507,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_5_4_TYPE_DTO = ColumnTypeDto.YEAR; public final static Long COLUMN_5_4_DATE_FORMAT = null; public final static Boolean COLUMN_5_4_NULL = true; - public final static Boolean COLUMN_5_4_UNIQUE = false; public final static Boolean COLUMN_5_4_AUTO_GENERATED = false; public final static String COLUMN_5_4_FOREIGN_KEY = null; public final static String COLUMN_5_4_CHECK = null; @@ -2533,7 +2524,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_5_5_TYPE_DTO = ColumnTypeDto.TIME; public final static Long COLUMN_5_5_DATE_FORMAT = null; public final static Boolean COLUMN_5_5_NULL = true; - public final static Boolean COLUMN_5_5_UNIQUE = false; public final static Boolean COLUMN_5_5_AUTO_GENERATED = false; public final static String COLUMN_5_5_FOREIGN_KEY = null; public final static String COLUMN_5_5_CHECK = null; @@ -2551,7 +2541,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_5_6_TYPE_DTO = ColumnTypeDto.BIGINT; public final static Long COLUMN_5_6_DATE_FORMAT = null; public final static Boolean COLUMN_5_6_NULL = true; - public final static Boolean COLUMN_5_6_UNIQUE = false; public final static Boolean COLUMN_5_6_AUTO_GENERATED = false; public final static String COLUMN_5_6_FOREIGN_KEY = null; public final static String COLUMN_5_6_CHECK = null; @@ -2569,7 +2558,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_8_1_TYPE_DTO = ColumnTypeDto.BIGINT; public final static Long COLUMN_8_1_DATE_FORMAT = null; public final static Boolean COLUMN_8_1_NULL = false; - public final static Boolean COLUMN_8_1_UNIQUE = true; public final static Boolean COLUMN_8_1_AUTO_GENERATED = true; public final static String COLUMN_8_1_FOREIGN_KEY = null; public final static String COLUMN_8_1_CHECK = null; @@ -2587,7 +2575,6 @@ public abstract class BaseTest { public final static ColumnTypeDto COLUMN_8_2_TYPE_DTO = ColumnTypeDto.INT; public final static Long COLUMN_8_2_DATE_FORMAT = null; public final static Boolean COLUMN_8_2_NULL = true; - public final static Boolean COLUMN_8_2_UNIQUE = false; public final static Boolean COLUMN_8_2_AUTO_GENERATED = false; public final static String COLUMN_8_2_FOREIGN_KEY = null; public final static String COLUMN_8_2_CHECK = null; @@ -3105,9 +3092,11 @@ public abstract class BaseTest { .build()); public final static Long TABLE_1_FOREIGN_KEY_1_ID = 1L; + public final static String TABLE_1_FOREIGN_KEY_1_NAME = "FK_JUNIT_1"; public final static ForeignKey TABLE_1_FOREIGN_KEY_1 = ForeignKey.builder() .fkid(TABLE_1_FOREIGN_KEY_1_ID) + .name(TABLE_1_FOREIGN_KEY_1_NAME) .referencedTable(TABLE_2) .table(TABLE_1) .references(List.of()) /* TABLE_1_FOREIGN_KEY_REFERENCE */ @@ -3125,6 +3114,7 @@ public abstract class BaseTest { public final static Constraints TABLE_1_CONSTRAINTS = Constraints.builder() .foreignKeys(List.of(TABLE_1_FOREIGN_KEY_1)) .uniques(List.of(Unique.builder() + .name("UK_1") .columns(List.of( TABLE_1_COLUMNS.get(0), TABLE_1_COLUMNS.get(1) @@ -3136,6 +3126,7 @@ public abstract class BaseTest { public final static Constraints TABLE_2_CONSTRAINTS = Constraints.builder() .uniques(List.of(Unique.builder() + .name("UK_1") .columns(List.of(TABLE_2_COLUMNS.get(0))) .table(TABLE_2) .build())) @@ -4122,6 +4113,7 @@ public abstract class BaseTest { public final static Constraints TABLE_3_CONSTRAINTS = Constraints.builder() .uniques(List.of(Unique.builder() + .name("UK_1") .columns(List.of(TABLE_3_COLUMNS.get(0))) .table(TABLE_3) .build())) @@ -4638,7 +4630,10 @@ public abstract class BaseTest { .build()); public final static Constraints TABLE_5_CONSTRAINTS = Constraints.builder() - .uniques(List.of(Unique.builder().columns(List.of(TABLE_5_COLUMNS.get(0))).build())) + .uniques(List.of(Unique.builder() + .name("UK_1") + .columns(List.of(TABLE_5_COLUMNS.get(0))) + .build())) .build(); public final static List<ForeignKeyCreateDto> TABLE_5_FOREIGN_KEYS_INVALID_CREATE = List.of(ForeignKeyCreateDto.builder() @@ -4768,7 +4763,10 @@ public abstract class BaseTest { .build()); public final static Constraints TABLE_6_CONSTRAINTS = Constraints.builder() - .uniques(List.of(Unique.builder().columns(List.of(TABLE_6_COLUMNS.get(0))).build())) + .uniques(List.of(Unique.builder() + .name("UK_1") + .columns(List.of(TABLE_6_COLUMNS.get(0))) + .build())) .build(); public final static List<ColumnCreateDto> TABLE_6_COLUMNS_CREATE = List.of( @@ -4845,7 +4843,6 @@ public abstract class BaseTest { public final static TableColumnType COLUMN_6_1_TYPE = TableColumnType.BIGINT; public final static Long COLUMN_6_1_DATE_FORMAT = null; public final static Boolean COLUMN_6_1_NULL = false; - public final static Boolean COLUMN_6_1_UNIQUE = false; public final static Boolean COLUMN_6_1_AUTO_GENERATED = false; public final static String COLUMN_6_1_FOREIGN_KEY = null; public final static String COLUMN_6_1_CHECK = null; @@ -4862,7 +4859,6 @@ public abstract class BaseTest { public final static TableColumnType COLUMN_6_2_TYPE = TableColumnType.BIGINT; public final static Long COLUMN_6_2_DATE_FORMAT = null; public final static Boolean COLUMN_6_2_NULL = false; - public final static Boolean COLUMN_6_2_UNIQUE = false; public final static Boolean COLUMN_6_2_AUTO_GENERATED = false; public final static String COLUMN_6_2_FOREIGN_KEY = null; public final static String COLUMN_6_2_CHECK = null; @@ -4908,12 +4904,6 @@ public abstract class BaseTest { public final static String VIEW_1_QUERY = "select `location`, `lat`, `lng` from `weather_location`"; public final static String VIEW_1_QUERY_HASH = "dc81a6877c7c51a6a6f406e1fc2a255e44a0d49a20548596e0d583c3eb849c23"; - public final static List<TableColumn> VIEW_1_COLUMNS = List.of( - TABLE_2_COLUMNS.get(0), - TABLE_2_COLUMNS.get(1), - TABLE_2_COLUMNS.get(2) - ); - public final static List<ColumnDto> VIEW_1_COLUMNS_DTO = List.of( TABLE_2_COLUMNS_DTO.get(0), TABLE_2_COLUMNS_DTO.get(1), @@ -4930,9 +4920,30 @@ public abstract class BaseTest { .query(VIEW_1_QUERY) .queryHash(VIEW_1_QUERY_HASH) .createdBy(USER_1_ID) - .columns(VIEW_1_COLUMNS) + .columns(null) /* VIEW_1_COLUMNS */ .build(); + public final static List<ViewColumn> VIEW_1_COLUMNS = List.of( + ViewColumn.builder() + .id(1L) + .ordinalPosition(0) + .column(TABLE_2_COLUMNS.get(0)) + .view(VIEW_1) + .build(), + ViewColumn.builder() + .id(2L) + .ordinalPosition(1) + .column(TABLE_2_COLUMNS.get(1)) + .view(VIEW_1) + .build(), + ViewColumn.builder() + .id(3L) + .ordinalPosition(2) + .column(TABLE_2_COLUMNS.get(2)) + .view(VIEW_1) + .build() + ); + public final static ViewDto VIEW_1_DTO = ViewDto.builder() .id(VIEW_1_ID) .isInitialView(VIEW_1_INITIAL_VIEW) @@ -4971,16 +4982,9 @@ public abstract class BaseTest { public final static Long VIEW_2_CONTAINER_ID = CONTAINER_1_ID; public final static Long VIEW_2_DATABASE_ID = DATABASE_1_ID; public final static Boolean VIEW_2_PUBLIC = true; - public final static String VIEW_2_QUERY = "select `date`, `location` as loc, `rainfall`, `mintemp` from `weather_aus` where `location` = 'Albury'"; + public final static String VIEW_2_QUERY = "select `date`, `location` as loc, `location`, `rainfall`, `mintemp` from `weather_aus` where `location` = 'Albury'"; public final static String VIEW_2_QUERY_HASH = "987fc946772ffb6d85060262dcb5df419692a1f6772ea995e3dedb53c191e984"; - public final static List<TableColumn> VIEW_2_COLUMNS = List.of( - TABLE_1_COLUMNS.get(1), - TABLE_1_COLUMNS.get(2), - TABLE_1_COLUMNS.get(4), - TABLE_1_COLUMNS.get(3) - ); - public final static List<ColumnDto> VIEW_2_COLUMNS_DTO = List.of( TABLE_1_COLUMNS_DTO.get(1), TABLE_1_COLUMNS_DTO.get(2), @@ -4995,12 +4999,40 @@ public abstract class BaseTest { .internalName(VIEW_2_INTERNAL_NAME) .vdbid(VIEW_2_DATABASE_ID) .isPublic(VIEW_2_PUBLIC) - .columns(VIEW_2_COLUMNS) + .columns(null) /* VIEW_2_COLUMNS */ .query(VIEW_2_QUERY) .queryHash(VIEW_2_QUERY_HASH) .createdBy(USER_1_ID) .build(); + public final static List<ViewColumn> VIEW_2_COLUMNS = List.of( + ViewColumn.builder() + .id(4L) + .ordinalPosition(0) + .column(TABLE_1_COLUMNS.get(1)) + .view(VIEW_2) + .build(), + ViewColumn.builder() + .id(5L) + .ordinalPosition(1) + .alias("loc") + .column(TABLE_1_COLUMNS.get(2)) + .view(VIEW_2) + .build(), + ViewColumn.builder() + .id(6L) + .ordinalPosition(2) + .column(TABLE_1_COLUMNS.get(4)) + .view(VIEW_2) + .build(), + ViewColumn.builder() + .id(7L) + .ordinalPosition(3) + .column(TABLE_1_COLUMNS.get(3)) + .view(VIEW_2) + .build() + ); + public final static ViewDto VIEW_2_DTO = ViewDto.builder() .id(VIEW_2_ID) .isInitialView(VIEW_2_INITIAL_VIEW) @@ -5036,13 +5068,6 @@ public abstract class BaseTest { public final static String VIEW_3_QUERY = "select w.`mintemp`, w.`rainfall`, w.`location`, m.`date` from `weather_aus` w join `junit2` m on m.`location` = w.`location` and m.`date` = w.`date`"; public final static String VIEW_3_QUERY_HASH = "bbbaa56a5206b3dc3e6cf9301b0db9344eb6f19b100c7b88550ffb597a0bd255"; - public final static List<TableColumn> VIEW_3_COLUMNS = List.of( - TABLE_1_COLUMNS.get(3), - TABLE_1_COLUMNS.get(4), - TABLE_1_COLUMNS.get(2), - TABLE_1_COLUMNS.get(1) - ); - public final static List<ColumnDto> VIEW_3_COLUMNS_DTO = List.of( TABLE_1_COLUMNS_DTO.get(3), TABLE_1_COLUMNS_DTO.get(4), @@ -5057,12 +5082,39 @@ public abstract class BaseTest { .internalName(VIEW_3_INTERNAL_NAME) .vdbid(VIEW_3_DATABASE_ID) .isPublic(VIEW_3_PUBLIC) - .columns(VIEW_3_COLUMNS) + .columns(null) /* VIEW_3_COLUMNS */ .query(VIEW_3_QUERY) .queryHash(VIEW_3_QUERY_HASH) .createdBy(USER_1_ID) .build(); + public final static List<ViewColumn> VIEW_3_COLUMNS = List.of( + ViewColumn.builder() + .id(8L) + .ordinalPosition(0) + .column(TABLE_1_COLUMNS.get(3)) + .view(VIEW_3) + .build(), + ViewColumn.builder() + .id(9L) + .ordinalPosition(1) + .column(TABLE_1_COLUMNS.get(4)) + .view(VIEW_3) + .build(), + ViewColumn.builder() + .id(10L) + .ordinalPosition(2) + .column(TABLE_1_COLUMNS.get(2)) + .view(VIEW_3) + .build(), + ViewColumn.builder() + .id(11L) + .ordinalPosition(3) + .column(TABLE_1_COLUMNS.get(1)) + .view(VIEW_3) + .build() + ); + public final static ViewDto VIEW_3_DTO = ViewDto.builder() .id(VIEW_3_ID) .isInitialView(VIEW_3_INITIAL_VIEW) @@ -5100,26 +5152,6 @@ public abstract class BaseTest { public final static String VIEW_4_QUERY = "SELECT `animal_name`, `hair`, `feathers`, `eggs`, `milk`, `airborne`, `aquatic`, `predator`, `backbone`, `breathes`, `venomous`, `fins`, `legs`, `tail`, `domestic`, `catsize`, `class_type` FROM `zoo` WHERE `class_type` = 1"; public final static String VIEW_4_QUERY_HASH = "3561cd0bb0b0e94d6f15ae602134252a5760d09d660a71a4fb015b6991c8ba0b"; - public final static List<TableColumn> VIEW_4_COLUMNS = List.of( - TABLE_5_COLUMNS.get(1), - TABLE_5_COLUMNS.get(2), - TABLE_5_COLUMNS.get(3), - TABLE_5_COLUMNS.get(5), - TABLE_5_COLUMNS.get(6), - TABLE_5_COLUMNS.get(8), - TABLE_5_COLUMNS.get(10), - TABLE_5_COLUMNS.get(11), - TABLE_5_COLUMNS.get(12), - TABLE_5_COLUMNS.get(13), - TABLE_5_COLUMNS.get(14), - TABLE_5_COLUMNS.get(15), - TABLE_5_COLUMNS.get(16), - TABLE_5_COLUMNS.get(17), - TABLE_5_COLUMNS.get(18), - TABLE_5_COLUMNS.get(19), - TABLE_5_COLUMNS.get(20) - ); - public final static List<ColumnDto> VIEW_4_COLUMNS_DTO = List.of( TABLE_5_COLUMNS_DTO.get(1), TABLE_5_COLUMNS_DTO.get(2), @@ -5150,9 +5182,114 @@ public abstract class BaseTest { .query(VIEW_4_QUERY) .queryHash(VIEW_4_QUERY_HASH) .createdBy(USER_1_ID) - .columns(VIEW_4_COLUMNS) + .columns(null) /* VIEW_4_COLUMNS */ .build(); + public final static List<ViewColumn> VIEW_4_COLUMNS = List.of( + ViewColumn.builder() + .id(12L) + .ordinalPosition(0) + .column(TABLE_5_COLUMNS.get(1)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(13L) + .ordinalPosition(1) + .column(TABLE_5_COLUMNS.get(2)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(14L) + .ordinalPosition(2) + .column(TABLE_5_COLUMNS.get(3)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(15L) + .ordinalPosition(3) + .column(TABLE_5_COLUMNS.get(5)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(16L) + .ordinalPosition(4) + .column(TABLE_5_COLUMNS.get(6)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(17L) + .ordinalPosition(5) + .column(TABLE_5_COLUMNS.get(8)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(18L) + .ordinalPosition(6) + .column(TABLE_5_COLUMNS.get(10)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(19L) + .ordinalPosition(7) + .column(TABLE_5_COLUMNS.get(11)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(20L) + .ordinalPosition(8) + .column(TABLE_5_COLUMNS.get(12)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(21L) + .ordinalPosition(9) + .column(TABLE_5_COLUMNS.get(13)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(22L) + .ordinalPosition(10) + .column(TABLE_5_COLUMNS.get(14)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(23L) + .ordinalPosition(11) + .column(TABLE_5_COLUMNS.get(15)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(24L) + .ordinalPosition(12) + .column(TABLE_5_COLUMNS.get(16)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(25L) + .ordinalPosition(13) + .column(TABLE_5_COLUMNS.get(17)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(26L) + .ordinalPosition(14) + .column(TABLE_5_COLUMNS.get(18)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(27L) + .ordinalPosition(15) + .column(TABLE_5_COLUMNS.get(19)) + .view(VIEW_4) + .build(), + ViewColumn.builder() + .id(28L) + .ordinalPosition(16) + .column(TABLE_5_COLUMNS.get(20)) + .view(VIEW_4) + .build() + ); + public final static ViewDto VIEW_4_DTO = ViewDto.builder() .id(VIEW_4_ID) .isInitialView(VIEW_4_INITIAL_VIEW) @@ -5186,6 +5323,7 @@ public abstract class BaseTest { .query(VIEW_5_QUERY) .queryHash(VIEW_5_QUERY_HASH) .createdBy(USER_1_ID) + .columns(null) .build(); public final static Long QUERY_1_RESULT_ID = 1L; diff --git a/dbrepo-ui/README.md b/dbrepo-ui/README.md index fbd83426ca01b7b4acb8aad17690bf577cd73945..5717916121eedacd881ee913a6e7fb9b2d4e9bb6 100644 --- a/dbrepo-ui/README.md +++ b/dbrepo-ui/README.md @@ -40,4 +40,15 @@ Optionally, generate a coverage report: ```bash yarn run coverage -``` \ No newline at end of file +``` + +## Troubleshooting + +Watchpack Error (watcher): Error: ENOSPC: System limit for number of file watchers reached, +watch `./dbrepo-ui/node_modules/...` + +* Cause: Started the local development server with `yarn dev` and the system file watchers could not be created since + the maximum limit is reached, debug with `cat /proc/sys/fs/inotify/max_user_watches`. +* Solution: Increase the limit with `sudo sysctl fs.inotify.max_user_watches=131070` and verify + success: `sudo sysctl -p` +* See further: [https://stackoverflow.com/questions/53930305/nodemon-error-system-limit-for-number-of-file-watchers-reached](https://stackoverflow.com/questions/53930305/nodemon-error-system-limit-for-number-of-file-watchers-reached) diff --git a/dbrepo-ui/pages/database/_database_id/table/_table_id/data.vue b/dbrepo-ui/pages/database/_database_id/table/_table_id/data.vue index c663a38827699e4d6bb262d9e4db8979ae6e2298..d19a2525c5e1956d253f62f34b49b1ad35998679 100644 --- a/dbrepo-ui/pages/database/_database_id/table/_table_id/data.vue +++ b/dbrepo-ui/pages/database/_database_id/table/_table_id/data.vue @@ -249,7 +249,7 @@ export default { this.table.columns.map((c) => { return { value: c.internal_name, - text: c.name, + text: c.internal_name, sortable: false } }).forEach(header => this.headers.push(header)) diff --git a/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue b/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue index 980b440ddd6beaa3a89283e350f5b6a657bc88da..331ca955fa666d43abc2868d2c318b9cb9022711 100644 --- a/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue +++ b/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue @@ -54,6 +54,16 @@ label="Value quotes" /> </v-col> </v-row> + <v-row dense> + <v-col cols="8"> + <v-text-field + v-model="tableImport.line_termination" + hint="Representation of a new line" + placeholder="e.g. \r\n" + clearable + label="Line termination" /> + </v-col> + </v-row> <v-row dense> <v-col cols="8"> <v-text-field @@ -143,8 +153,9 @@ export default { quote: '"', false_element: null, true_element: null, - null_element: 'NA', + null_element: '', separator: ',', + line_termination: '\\r\\n', skip_lines: 1 }, file: { diff --git a/dbrepo-ui/pages/database/_database_id/table/_table_id/schema.vue b/dbrepo-ui/pages/database/_database_id/table/_table_id/schema.vue index ccddb4a1d8cf971268ad1d8b4c2da547ec7a110d..6b79b38ec344755ea56b6518c6594c2e4d1baeab 100644 --- a/dbrepo-ui/pages/database/_database_id/table/_table_id/schema.vue +++ b/dbrepo-ui/pages/database/_database_id/table/_table_id/schema.vue @@ -3,7 +3,7 @@ <TableToolbar :selection="selection" /> <v-toolbar color="secondary white--text" flat> <strong> - <v-toolbar-title v-text="title" /> + <v-toolbar-title>Schema</v-toolbar-title> </strong> </v-toolbar> <v-card tile> @@ -64,6 +64,29 @@ </template> </v-data-table> </v-card> + <v-card v-if="hasConstraints" tile> + <v-card-subtitle>Constraints</v-card-subtitle> + <v-card-text> + <ul> + <li v-for="(foreignKey,i) in table.constraints.foreign_keys" :key="`fk-${i}`"> + <strong>FOREIGN KEY</strong> + <span v-text="foreignKey.name" /> + (<i v-text="foreignKeyColumns(foreignKey)" />) + <strong>REFERENCES</strong> + <a :href="`/database/${database.id}/table/${foreignKey.referenced_table.id}/schema`" v-text="foreignKeyReferencedTable(foreignKey)" /> + (<i v-text="foreignKeyReferencedColumns(foreignKey)" />) + </li> + <li v-for="(uniqueConstraint,i) in table.constraints.uniques" :key="`uk-${i}`"> + <strong>UNIQUE INDEX</strong> + (<i v-text="uniqueColumns(uniqueConstraint)" />) + </li> + <li v-for="(checkConstraint,i) in table.constraints.checks" :key="`uk-${i}`"> + <strong>CHECK CONSTRAINT</strong> + (<i v-text="checkConstraint" />) + </li> + </ul> + </v-card-text> + </v-card> <v-dialog v-if="table && database" v-model="dialogSemantic" @@ -81,7 +104,6 @@ </template> <script> import TableToolbar from '@/components/table/TableToolbar.vue' -import TableService from '@/api/table.service' export default { components: { @@ -106,7 +128,6 @@ export default { { value: 'column_concept', text: 'Concept' }, { value: 'column_unit', text: 'Unit' }, { value: 'is_primary_key', text: 'Primary Key' }, - { value: 'unique', text: 'Unique' }, { value: 'is_null_allowed', text: 'Nullable' }, { value: 'auto_generated', text: 'Sequence' } ], @@ -129,12 +150,6 @@ export default { roles () { return this.$store.state.roles }, - title () { - if (!this.table) { - return null - } - return this.table.constraints.checks.length > 0 ? `Constraints: ${this.table.constraints.checks}` : 'Schema' - }, canAssignSemanticInformation () { if (!this.user) { return false @@ -147,23 +162,11 @@ export default { } return this.roles.includes('modify-table-column-semantics') && (this.access.type === 'write_all' || this.table.owner.username === this.user.username) }, - versionColor () { - if (this.version === null) { - return 'secondary white--text' - } - return 'primary white--text' - }, - versionFormatted () { - if (this.version === null) { - return null - } - return this.version + ' (UTC)' - }, - versionISO () { - if (this.version === null) { - return null + hasConstraints () { + if (!this.table || !this.table.constraints) { + return false } - return this.version.substring(0, 10) + 'T' + this.version.substring(11, 19) + 'Z' + return this.table.constraints.uniques.length > 0 || this.table.constraints.checks.length > 0 || this.table.constraints.foreign_keys.length > 0 } }, mounted () { @@ -177,13 +180,6 @@ export default { const uniqueColumnIds = this.table.constraints.uniques.map(u => u.columns.map(c => c.id)).flat() return uniqueColumnIds.includes(column.id) }, - columnName (column) { - const filter = this.columnTypes.filter(t => t.value === column.column_type) - if (filter.length > 0) { - return filter[0].text - } - return column.column_type - }, extra (column) { if (['date', 'datetime', 'timestamp', 'time'].includes(column.column_type)) { return `fsp=${column.date_format.unix_format}` @@ -219,21 +215,29 @@ export default { } this.dialogSemantic = false }, - loadTable () { - if (!this.$route.params.database_id || !this.$route.params.table_id) { - return + foreignKeyColumns (foreignKey) { + if (!foreignKey) { + return null + } + return foreignKey.columns.map(c => c.internal_name).join(',') + }, + foreignKeyReferencedTable (foreignKey) { + if (!foreignKey) { + return null + } + return foreignKey.referenced_table.internal_name + }, + foreignKeyReferencedColumns (foreignKey) { + if (!foreignKey) { + return null + } + return foreignKey.referenced_columns.map(c => c.internal_name).join(',') + }, + uniqueColumns (uniqueConstraint) { + if (!uniqueConstraint) { + return null } - this.loading = true - TableService.findOne(this.$route.params.database_id, this.$route.params.table_id) - .then((table) => { - this.$store.commit('SET_TABLE', table) - }) - .catch(() => { - this.loading = false - }) - .finally(() => { - this.loading = false - }) + return uniqueConstraint.columns.map(c => c.internal_name).join(',') } } } diff --git a/dbrepo-ui/pages/database/_database_id/table/import.vue b/dbrepo-ui/pages/database/_database_id/table/import.vue index a013fcff66234e4bf2a5348714140e35a5263944..4faa62a52079bf686674cf54f3123bfcb0a25a60 100644 --- a/dbrepo-ui/pages/database/_database_id/table/import.vue +++ b/dbrepo-ui/pages/database/_database_id/table/import.vue @@ -88,6 +88,16 @@ label="Value quotes" /> </v-col> </v-row> + <v-row dense> + <v-col cols="8"> + <v-text-field + v-model="tableImport.line_termination" + hint="Representation of a new line" + placeholder="e.g. \r\n" + clearable + label="Line termination" /> + </v-col> + </v-row> <v-row dense> <v-col cols="8"> <v-text-field @@ -262,8 +272,9 @@ export default { quote: '"', false_element: null, true_element: null, - null_element: 'NA', + null_element: '', separator: ',', + line_termination: '\\r\\n', skip_lines: 1 }, loading: false, diff --git a/helm-charts/dbrepo/Chart.yaml b/helm-charts/dbrepo/Chart.yaml index 79a5fd1fa2cac445afa3c7d661a17589c7999129..fe4c5cf04a7f1637d6fff38a3311851131a9ce92 100644 --- a/helm-charts/dbrepo/Chart.yaml +++ b/helm-charts/dbrepo/Chart.yaml @@ -16,7 +16,7 @@ home: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/ icon: https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-services/-/raw/master/.docs/images/signet_white.png dependencies: - name: opensearch - alias: searchDb + alias: searchdb version: 2.15.0 # app version 2.10.0 repository: https://opensearch-project.github.io/helm-charts/ - name: opensearch-dashboards @@ -44,10 +44,10 @@ dependencies: version: 12.5.1 repository: https://charts.bitnami.com/bitnami - name: fluent-bit - alias: logService + alias: logservice version: 0.40.0 repository: https://fluent.github.io/helm-charts - name: seaweedfs - alias: storageService + alias: storageservice version: 3.59.4 repository: https://seaweedfs.github.io/seaweedfs/helm diff --git a/helm-charts/dbrepo/README.md b/helm-charts/dbrepo/README.md index bfd84c76f8626453ddafa56e703b5e42f8c64327..d7af72f404d5d894ce0965435be4ebbbef6c1138 100644 --- a/helm-charts/dbrepo/README.md +++ b/helm-charts/dbrepo/README.md @@ -58,30 +58,30 @@ The command removes all the Kubernetes components associated with the chart and The Metadata Database uses the [Bitnami MariaDB Galera](https://artifacthub.io/packages/helm/bitnami/mariadb-galera) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|-----------------------------|-------------------------------------------|---------------| -| `metadata-db.host` | Hostname. | `metadata-db` | -| `metadata-db.jdbcExtraArgs` | Extra arguments for the JDBC connections. | `""` | +| Name | Description | Value | +|----------------------------|-------------------------------------------|---------------| +| `metadataDb.host` | Hostname. | `metadata-db` | +| `metadataDb.jdbcExtraArgs` | Extra arguments for the JDBC connections. | `""` | ### Authentication Service The Auth Service uses the [Bitnami Keycloak](https://artifacthub.io/packages/helm/bitnami/keycloak) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|------------------------------|-----------------------------------------------------------------|------------------------------------| -| `auth-service.client.id` | Client id. This value is publicly known. | `dbrepo-client` | -| `auth-service.client.secret` | Client secret. This value should never be known outside DBRepo. | `MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG` | +| Name | Description | Value | +|-----------------------------|-----------------------------------------------------------------|------------------------------------| +| `authService.client.id` | Client id. This value is publicly known. | `dbrepo-client` | +| `authService.client.secret` | Client secret. This value should never be known outside DBRepo. | `MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG` | ### Auth Database The Auth Database uses the [Bitnami PostgreSQL HA](https://artifacthub.io/packages/helm/bitnami/postgresql-ha) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|----------------|--------------------------------------|------------------| -| `auth-db.host` | Hostname. Needed for other services. | `auth-db-pgpool` | -| `auth-db.port` | Port. Needed for other services. | `5432` | +| Name | Description | Value | +|---------------|--------------------------------------|------------------| +| `authDb.host` | Hostname. Needed for other services. | `auth-db-pgpool` | +| `authDb.port` | Port. Needed for other services. | `5432` | ### Data Database @@ -95,12 +95,12 @@ The Search Database uses the [OpenSearch](https://artifacthub.io/packages/helm/opensearch-project-helm-charts/opensearch) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|----------------------|--------------------------------------|-------------| -| `search-db.host` | Hostname. Needed for other services. | `search-db` | -| `search-db.port` | Port. Needed for other services. | `9200` | -| `search-db.username` | Username. Needed for other services. | `admin` | -| `search-db.password` | Password. Needed for other services. | `admin` | +| Name | Description | Value | +|---------------------|--------------------------------------|-------------| +| `searchdb.host` | Hostname. Needed for other services. | `search-db` | +| `searchdb.port` | Port. Needed for other services. | `9200` | +| `searchdb.username` | Username. Needed for other services. | `admin` | +| `searchdb.password` | Password. Needed for other services. | `admin` | ### Search Database Dashboard @@ -110,29 +110,29 @@ chart. See their documentation for the remaining overridden values. ### Upload Service -| Name | Description | Value | -|-----------------------------------|----------------------------------------|-------------------| -| `upload-service.enabled` | Enables/disabled the deployment. | `true` | -| `upload-service.image.registry` | Registry to pull the image | `docker.io` | -| `upload-service.image.repository` | Repository to pull the image | `tusproject/tusd` | -| `upload-service.image.tag` | Tag of the image. | `v1.12` | -| `upload-service.replicaCount` | Number of replicas for the deployment. | `2` | +| Name | Description | Value | +|----------------------------------|----------------------------------------|-------------------| +| `uploadService.enabled` | Enables/disabled the deployment. | `true` | +| `uploadService.image.registry` | Registry to pull the image | `docker.io` | +| `uploadService.image.repository` | Repository to pull the image | `tusproject/tusd` | +| `uploadService.image.tag` | Tag of the image. | `v1.12` | +| `uploadService.replicaCount` | Number of replicas for the deployment. | `2` | ### Broker Service The Broker Service uses the [Bitnami RabbitMQ](https://artifacthub.io/packages/helm/bitnami/rabbitmq) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|------------------------------------|-------------------------------------------------------------------------|-------------------------------| -| `broker-service.url` | Admin API endpoint. Needed for other services. | `http://broker-service:15672` | -| `broker-service.host` | Service hostname. Needed for other services. | `broker-service` | -| `broker-service.port` | Service port. Needed for other services. | `5672` | -| `broker-service.virtualHost` | Virtual host on RabbitMQ. Needed for other services. | `dbrepo` | -| `broker-service.queueName` | Queue name on RabbitMQ. Needed for other services. | `dbrepo` | -| `broker-service.exchangeName` | Exchange name on RabbitMQ. Needed for other services. | `dbrepo` | -| `broker-service.routingKey` | Route binding for queue to exchange defined. Needed for other services. | `dbrepo.#` | -| `broker-service.connectionTimeout` | Connection timeout. Needed for other services. | `60000` | +| Name | Description | Value | +|-----------------------------------|-------------------------------------------------------------------------|-------------------------------| +| `brokerService.url` | Admin API endpoint. Needed for other services. | `http://broker-service:15672` | +| `brokerService.host` | Service hostname. Needed for other services. | `broker-service` | +| `brokerService.port` | Service port. Needed for other services. | `5672` | +| `brokerService.virtualHost` | Virtual host on RabbitMQ. Needed for other services. | `dbrepo` | +| `brokerService.queueName` | Queue name on RabbitMQ. Needed for other services. | `dbrepo` | +| `brokerService.exchangeName` | Exchange name on RabbitMQ. Needed for other services. | `dbrepo` | +| `brokerService.routingKey` | Route binding for queue to exchange defined. Needed for other services. | `dbrepo.#` | +| `brokerService.connectionTimeout` | Connection timeout. Needed for other services. | `60000` | ### Analyse Service @@ -204,10 +204,10 @@ Helm chart. See their documentation for the remaining overridden values. The Storage Service uses the [SeaweedFS](https://artifacthub.io/packages/helm/seaweedfs/seaweedfs) Helm chart. See their documentation for the remaining overridden values. -| Name | Description | Value | -|---------------------------------|---------------------------------------------|------------------| -| `storage-service.auth.username` | Username for S3. Needed for other services. | `seaweedfsadmin` | -| `storage-service.auth.password` | Password for S3. Needed for other services. | `seaweedfsadmin` | +| Name | Description | Value | +|--------------------------------|---------------------------------------------|------------------| +| `storageservice.auth.username` | Username for S3. Needed for other services. | `seaweedfsadmin` | +| `storageservice.auth.password` | Password for S3. Needed for other services. | `seaweedfsadmin` | ### User Interface diff --git a/helm-charts/dbrepo/templates/auth-service/configmap.yaml b/helm-charts/dbrepo/templates/auth-service/configmap.yaml index da7f14e0e3056fac5421b12911dd026dd782f1f7..fcb56216b8d761ffe3eb7d140fa5fb59667b6613 100644 --- a/helm-charts/dbrepo/templates/auth-service/configmap.yaml +++ b/helm-charts/dbrepo/templates/auth-service/configmap.yaml @@ -106,7 +106,7 @@ data: "description" : "${default-database-handling}", "composite" : true, "composites" : { - "realm" : [ "modify-database-owner", "update-database-access", "create-database", "list-databases", "create-database-access", "find-database", "modify-database-visibility", "import-database-data", "delete-database-access", "check-database-access" ] + "realm" : [ "modify-database-image", "modify-database-owner", "update-database-access", "create-database", "list-databases", "create-database-access", "find-database", "modify-database-visibility", "import-database-data", "delete-database-access", "check-database-access" ] }, "clientRole" : false, "containerId" : "82c39861-d877-4667-a0f3-4daa2ee230e0", @@ -227,6 +227,14 @@ data: "clientRole" : false, "containerId" : "82c39861-d877-4667-a0f3-4daa2ee230e0", "attributes" : { } + }, { + "id" : "1f0a9b13-c2b8-474c-bc08-59dbd71835a6", + "name" : "modify-database-image", + "description" : "${modify-database-image}", + "composite" : false, + "clientRole" : false, + "containerId" : "82c39861-d877-4667-a0f3-4daa2ee230e0", + "attributes" : { } }, { "id" : "a7ad038c-5c06-42fc-951c-15ac09d4df66", "name" : "modify-database-owner", @@ -2056,7 +2064,7 @@ data: "subType" : "authenticated", "subComponents" : { }, "config" : { - "allowed-protocol-mapper-types" : [ "saml-role-list-mapper", "oidc-address-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-user-attribute-mapper", "saml-user-property-mapper", "oidc-full-name-mapper", "oidc-usermodel-attribute-mapper", "oidc-usermodel-property-mapper" ] + "allowed-protocol-mapper-types" : [ "oidc-sha256-pairwise-sub-mapper", "oidc-full-name-mapper", "saml-role-list-mapper", "oidc-address-mapper", "saml-user-attribute-mapper", "saml-user-property-mapper", "oidc-usermodel-attribute-mapper", "oidc-usermodel-property-mapper" ] } }, { "id" : "3ab11d74-5e76-408a-b85a-26bf8950f979", @@ -2065,7 +2073,7 @@ data: "subType" : "anonymous", "subComponents" : { }, "config" : { - "allowed-protocol-mapper-types" : [ "oidc-usermodel-property-mapper", "saml-user-property-mapper", "oidc-full-name-mapper", "oidc-address-mapper", "saml-user-attribute-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-role-list-mapper", "oidc-usermodel-attribute-mapper" ] + "allowed-protocol-mapper-types" : [ "saml-user-attribute-mapper", "saml-role-list-mapper", "oidc-address-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-usermodel-property-mapper", "oidc-full-name-mapper", "oidc-usermodel-attribute-mapper", "saml-user-property-mapper" ] } } ], "org.keycloak.keys.KeyProvider" : [ { @@ -2117,7 +2125,7 @@ data: "internationalizationEnabled" : false, "supportedLocales" : [ ], "authenticationFlows" : [ { - "id" : "b8378805-a082-46a0-9e28-a1e5d4db7e41", + "id" : "05f92ecb-5a34-416a-a9a4-b4aeab2704c4", "alias" : "Account verification options", "description" : "Method with which to verity the existing account", "providerId" : "basic-flow", @@ -2139,7 +2147,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "2652bbd9-bd49-465c-8595-690099333bf7", + "id" : "e85f1d42-30c8-4878-ab0c-3cb9baaa308f", "alias" : "Authentication Options", "description" : "Authentication options.", "providerId" : "basic-flow", @@ -2168,7 +2176,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "967c3248-c2e9-45a9-b770-b02e965b958a", + "id" : "754e6269-c096-41d6-88df-44bd2652ec82", "alias" : "Browser - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2190,7 +2198,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "f78ad348-c3e1-476e-a916-fce0c383376a", + "id" : "5b2a16dd-7192-4558-931a-a67dfa7b14e1", "alias" : "Direct Grant - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2212,7 +2220,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "788cf02b-5744-4ea6-940a-96bc762da4bd", + "id" : "c12d7c33-256e-486f-8fb8-c8594eafd64e", "alias" : "First broker login - Conditional OTP", "description" : "Flow to determine if the OTP is required for the authentication", "providerId" : "basic-flow", @@ -2234,7 +2242,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "273e61b7-9cc3-464e-a7b8-27c71aca4014", + "id" : "711adf58-692f-4f22-ae20-0ba01d8d667c", "alias" : "Handle Existing Account", "description" : "Handle what to do if there is existing account with same email/username like authenticated identity provider", "providerId" : "basic-flow", @@ -2256,7 +2264,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "00f41bfc-8513-466d-8c6a-366b7f2f36ca", + "id" : "dd53182d-ca4a-4096-b1fc-60237af977c4", "alias" : "Reset - Conditional OTP", "description" : "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.", "providerId" : "basic-flow", @@ -2278,7 +2286,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "980ebf01-fe0a-4cfa-880e-dd86ce8e190e", + "id" : "23c368c2-dce4-4ca8-8096-b6c726fa0e32", "alias" : "User creation or linking", "description" : "Flow for the existing/non-existing user alternatives", "providerId" : "basic-flow", @@ -2301,7 +2309,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "5e6a7a10-4be8-4038-8fc5-0588b452328d", + "id" : "37ff6b93-bdfe-4245-9247-009061fdfc7b", "alias" : "Verify Existing Account by Re-authentication", "description" : "Reauthentication of existing account", "providerId" : "basic-flow", @@ -2323,7 +2331,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "024e07f8-f975-41ef-b755-d2b089b5567c", + "id" : "c1f58e18-5d41-40b1-aa73-4a4e4a970430", "alias" : "browser", "description" : "browser based authentication", "providerId" : "basic-flow", @@ -2359,7 +2367,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "324da9be-755e-4556-a1d3-58569b9df47c", + "id" : "9229472e-78c8-4e83-aa20-7a2e22c28f59", "alias" : "clients", "description" : "Base authentication for clients", "providerId" : "client-flow", @@ -2395,7 +2403,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "bced47d4-5d04-4bb9-8605-94041185c0f3", + "id" : "d841dca1-b9ca-47bc-8f9a-dcd5896678dd", "alias" : "direct grant", "description" : "OpenID Connect Resource Owner Grant", "providerId" : "basic-flow", @@ -2424,7 +2432,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "6b301d9d-68c0-44c3-9a57-92669d08b2f3", + "id" : "42e0301c-d81c-4127-9e17-064811566f9a", "alias" : "docker auth", "description" : "Used by Docker clients to authenticate against the IDP", "providerId" : "basic-flow", @@ -2439,7 +2447,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "9c9ddfeb-37a2-4186-a58f-cf90dca8e191", + "id" : "4809629a-0e3c-4894-8cd7-60d99abeb2e8", "alias" : "first broker login", "description" : "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account", "providerId" : "basic-flow", @@ -2462,7 +2470,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "a9ef5094-93bf-49fc-9d0f-dcfc551cac5a", + "id" : "7ce37ac0-9aba-412d-98fb-78745e6df1ff", "alias" : "forms", "description" : "Username, password, otp and other auth forms.", "providerId" : "basic-flow", @@ -2484,7 +2492,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "fae6e2e4-a071-458b-ac03-41dda3456f5a", + "id" : "9fa4ee30-9ab4-40c3-bb9f-b56b8738d1c0", "alias" : "http challenge", "description" : "An authentication flow based on challenge-response HTTP Authentication Schemes", "providerId" : "basic-flow", @@ -2506,7 +2514,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "ae5bcac5-8867-42e1-887f-fc67418b0c4b", + "id" : "bba37884-4bd0-4597-9f26-e8b8c7d60dc6", "alias" : "registration", "description" : "registration flow", "providerId" : "basic-flow", @@ -2522,7 +2530,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "72524b5d-1cfc-41b0-b29b-6f6890d2dc7f", + "id" : "9e3b3ba5-e37e-4f6d-a7a7-fd37558f6e2d", "alias" : "registration form", "description" : "registration form", "providerId" : "form-flow", @@ -2558,7 +2566,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "834c96b8-790d-4869-8c66-d42cd35e4873", + "id" : "e38d574a-2171-408b-9f9d-1ebe60791110", "alias" : "reset credentials", "description" : "Reset credentials for a user if they forgot their password or something", "providerId" : "basic-flow", @@ -2594,7 +2602,7 @@ data: "userSetupAllowed" : false } ] }, { - "id" : "7f131501-e3ff-48f2-98e6-e34e4c5d6f9e", + "id" : "5560dfff-822c-43fb-a910-db38b4470268", "alias" : "saml ecp", "description" : "SAML ECP Profile Authentication Flow", "providerId" : "basic-flow", @@ -2610,13 +2618,13 @@ data: } ] } ], "authenticatorConfig" : [ { - "id" : "638341f1-94ba-4042-a3ee-41a0f41718f6", + "id" : "201f18f6-b170-4fcc-bcc2-2ca05b1558aa", "alias" : "create unique user config", "config" : { "require.password.update.after.registration" : "false" } }, { - "id" : "3c355b8c-8a51-4346-88f2-1ff81856b55c", + "id" : "f6e84d09-4994-452a-be1a-fe896289ae9d", "alias" : "review profile config", "config" : { "update.profile.on.first.login" : "missing" diff --git a/helm-charts/dbrepo/templates/data-service/secret.yaml b/helm-charts/dbrepo/templates/data-service/secret.yaml index 3386171d49f7e942130842cbfdb1864125fe6c27..8bfa60c1eaa069e517b23670f6e400ded2639b03 100644 --- a/helm-charts/dbrepo/templates/data-service/secret.yaml +++ b/helm-charts/dbrepo/templates/data-service/secret.yaml @@ -12,8 +12,8 @@ stringData: metadata-username: "{{ .Values.metadataDb.rootUser.user }}" metadata-password: "{{ .Values.metadataDb.rootUser.password }}" metadata-jdbc-extra-args: "{{ .Values.metadataDb.jdbcExtraArgs }}" - search-username: "{{ .Values.searchDb.username }}" - search-password: "{{ .Values.searchDb.password }}" + search-username: "{{ .Values.searchdb.username }}" + search-password: "{{ .Values.searchdb.password }}" jwt-issuer: "{{ $jwtIssuer }}" jwt-pubkey: "{{ .Values.dataService.jwt.pubkey }}" broker-username: "{{ .Values.brokerService.auth.username }}" diff --git a/helm-charts/dbrepo/templates/metadata-service/secret.yaml b/helm-charts/dbrepo/templates/metadata-service/secret.yaml index 374381bab36c01217c0ae4add61307df53baff3d..0976f3b63c1d5148590718b155ae4b3fbc31c977 100644 --- a/helm-charts/dbrepo/templates/metadata-service/secret.yaml +++ b/helm-charts/dbrepo/templates/metadata-service/secret.yaml @@ -9,25 +9,25 @@ metadata: stringData: admin-email: "{{ .Values.metadataService.adminEmail }}" base-url: "{{ .Values.hostname }}" - broker-endpoint: "{{ index .Values "broker-service" "url" }}" - broker-host: "{{ index .Values "broker-service" "host" }}" - broker-port: "{{ index .Values "broker-service" "port" }}" + broker-endpoint: "{{ .Values.brokerService.url }}" + broker-host: "{{ .Values.brokerService.host }}" + broker-port: "{{ .Values.brokerService.port }}" gateway-endpoint: "{{ .Values.hostname }}" website: "{{ .Values.metadataService.website }}" - search-username: "{{ index .Values "search-db" "username" }}" - search-password: "{{ index .Values "search-db" "password" }}" - broker-username: "{{ index .Values "broker-service" "auth" "username" }}" - broker-password: "{{ index .Values "broker-service" "auth" "password" }}" + search-username: "{{ .Values.searchdb.username }}" + search-password: "{{ .Values.searchdb.password }}" + broker-username: "{{ .Values.brokerService.auth.username }}" + broker-password: "{{ .Values.brokerService.auth.password }}" log-level: "{{ ternary "trace" "info" .Values.metadataService.image.debug }}" - metadata-db: "{{ index .Values "metadata-db" "db" "name" }}" - metadata-host: "{{ index .Values "metadata-db" "host" }}" - metadata-username: "{{ index .Values "metadata-db" "rootUser" "user" }}" - metadata-password: "{{ index .Values "metadata-db" "rootUser" "password" }}" - metadata-jdbc-extra-args: "{{ index .Values "metadata-db" "jdbcExtraArgs" }}" + metadata-db: "{{ .Values.metadataDb.db.name }}" + metadata-host: "{{ .Values.metadataDb.host }}" + metadata-username: "{{ .Values.metadataDb.rootUser.user }}" + metadata-password: "{{ .Values.metadataDb.rootUser.password }}" + metadata-jdbc-extra-args: "{{ .Values.metadataDb.jdbcExtraArgs }}" keycloak-host: "{{ .Values.metadataService.authService.url }}" - keycloak-admin: "{{ index .Values "auth-service" "auth" "adminUser" }}" - keycloak-admin-password: "{{ index .Values "auth-service" "auth" "adminPassword" }}" - keycloak-client-secret: "{{ index .Values "auth-service" "client" "secret" }}" + keycloak-admin: "{{ .Values.authService.auth.adminUser }}" + keycloak-admin-password: "{{ .Values.authService.auth.adminPassword }}" + keycloak-client-secret: "{{ .Values.authService.client.secret }}" datacite-url: "{{ .Values.metadataService.datacite.url }}" datacite-prefix: "{{ .Values.metadataService.datacite.prefix | toString }}" datacite-username: "{{ .Values.metadataService.datacite.username }}" @@ -35,17 +35,17 @@ stringData: repository-name: "{{ .Values.metadataService.repositoryName }}" pid-base: "{{ $pidBase }}" jwt-issuer: "{{ $jwtIssuer }}" - broker-virtualhost: "{{ index .Values "broker-service" "virtualHost" }}" - queue-name: "{{ index .Values "broker-service" "queueName" }}" - exchange-name: "{{ index .Values "broker-service" "exchangeName" }}" - routing-key: "{{ index .Values "broker-service" "routingKey" }}" - connection-timeout: "{{ index .Values "broker-service" "connectionTimeout" }}" + broker-virtualhost: "{{ .Values.brokerService.virtualHost }}" + queue-name: "{{ .Values.brokerService.queueName }}" + exchange-name: "{{ .Values.brokerService.exchangeName }}" + routing-key: "{{ .Values.brokerService.routingKey }}" + connection-timeout: "{{ .Values.brokerService.connectionTimeout }}" min-concurrent-consumers: "{{ .Values.dataService.consumerConcurrentMin }}" max-concurrent-consumers: "{{ .Values.dataService.consumerConcurrentMax }}" requeue-rejected: "{{ .Values.dataService.requeueRejected }}" s3-storage-endpoint: http://storage-service-s3:9000 - s3-access-key-id: "{{ index .Values "storage-service" "s3" "auth" "username" }}" - s3-secret-access-key: "{{ index .Values "storage-service" "s3" "auth" "password" }}" + s3-access-key-id: "{{ .Values.storageservice.s3.auth.username }}" + s3-secret-access-key: "{{ .Values.storageservice.s3.auth.password }}" s3-import-bucket: "dbrepo-upload" s3-export-bucket: "dbrepo-download" delete-stale-files-rate: {{ .Values.metadataService.rates.deleteStaleFiles | quote }} diff --git a/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml b/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml index f7caf1d29290312437cf014e2766d8b8eabcd226..9150e8e1d6b48d0168558fb2662ca04f2a1d74e6 100644 --- a/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml +++ b/helm-charts/dbrepo/templates/search-db-dashboard/secret.yaml @@ -17,5 +17,5 @@ stringData: opensearch: ssl: verificationMode: none - username: {{ .Values.searchDb.username }} - password: {{ .Values.searchDb.password }} + username: {{ .Values.searchdb.username }} + password: {{ .Values.searchdb.password }} diff --git a/helm-charts/dbrepo/templates/search-service/secret.yaml b/helm-charts/dbrepo/templates/search-service/secret.yaml index de636d510424d2250fe598463c4eb4b6b6be53b2..834c319e93f2219025e8c8dc27893561109e450c 100644 --- a/helm-charts/dbrepo/templates/search-service/secret.yaml +++ b/helm-charts/dbrepo/templates/search-service/secret.yaml @@ -5,8 +5,8 @@ metadata: name: search-service-secret namespace: {{ .Values.namespace }} stringData: - opensearch-host: "{{ .Values.searchDb.host }}" - opensearch-port: "{{ .Values.searchDb.port }}" - opensearch-username: "{{ .Values.searchDb.username }}" - opensearch-password: "{{ .Values.searchDb.password }}" + opensearch-host: "{{ .Values.searchdb.host }}" + opensearch-port: "{{ .Values.searchdb.port }}" + opensearch-username: "{{ .Values.searchdb.username }}" + opensearch-password: "{{ .Values.searchdb.password }}" log-level: "{{ ternary "DEBUG" "INFO" .Values.searchService.image.debug }}" diff --git a/helm-charts/dbrepo/templates/storage-service/secret.yaml b/helm-charts/dbrepo/templates/storage-service/secret.yaml index baf13a16ad937e93c364e6fbd4e1a594c10c227c..fb63db7f721a7fa356044df5c8aa41db7d6f9fd0 100644 --- a/helm-charts/dbrepo/templates/storage-service/secret.yaml +++ b/helm-charts/dbrepo/templates/storage-service/secret.yaml @@ -14,8 +14,8 @@ stringData: "name": "admin", "credentials": [ { - "accessKey": "{{ .Values.storageService.s3.auth.username }}", - "secretKey": "{{ .Values.storageService.s3.auth.password }}" + "accessKey": "{{ .Values.storageservice.s3.auth.username }}", + "secretKey": "{{ .Values.storageservice.s3.auth.password }}" } ], "actions": [ diff --git a/helm-charts/dbrepo/templates/upload-service/secret.yaml b/helm-charts/dbrepo/templates/upload-service/secret.yaml index ba309cc7ec41b7390f46c5e12d0a2893e31bee18..64d24c4396c7d6a84c2b6aef0da4b3d1c1043161 100644 --- a/helm-charts/dbrepo/templates/upload-service/secret.yaml +++ b/helm-charts/dbrepo/templates/upload-service/secret.yaml @@ -6,7 +6,7 @@ metadata: name: upload-service-secret namespace: {{ .Values.namespace }} stringData: - aws-access-key-id: "{{ .Values.storageService.s3.auth.username }}" - aws-secret-access-key: "{{ .Values.storageService.s3.auth.password }}" + aws-access-key-id: "{{ .Values.storageservice.s3.auth.username }}" + aws-secret-access-key: "{{ .Values.storageservice.s3.auth.password }}" aws-region: "default" {{- end }} \ No newline at end of file diff --git a/helm-charts/dbrepo/values.yaml b/helm-charts/dbrepo/values.yaml index ad8e6da8272de514cd310ea6ce2a0413054d4812..a31a9e8a7faee8a45884808e0047d8ca7b17a6c9 100644 --- a/helm-charts/dbrepo/values.yaml +++ b/helm-charts/dbrepo/values.yaml @@ -1,6 +1,6 @@ -namespace: +namespace: "" -hostname: +hostname: "" strategyType: RollingUpdate @@ -156,7 +156,7 @@ dataDb: sharedStorageClass: default replicaCount: 3 # uneven -searchDb: +searchdb: fullnameOverride: search-db host: search-db port: 9200 @@ -397,8 +397,7 @@ searchService: debug: false replicaCount: 2 -storageService: - fullnameOverride: storage-service +storageservice: master: enabled: true filer: @@ -428,7 +427,7 @@ storageService: username: seaweedfsadmin password: seaweedfsadmin -logService: +logservice: fullnameOverride: log-service config: outputs: | @@ -473,6 +472,7 @@ ui: { "title": "Database Repository", "version": "1.4.1", + "fluid": false, "ssl": { "force": false }, @@ -507,9 +507,9 @@ ui: } }, "upload": { - "endpoint": "localhost", - "port": 1080, - "useSsl": false + "endpoint": "upload-service", + "port": 80, + "useSsl": true }, "database": { "connection": {