diff --git a/.docs/system-services-metadata.md b/.docs/system-services-metadata.md
index 5c0e1d52001eac9f4c1833214a140aa7a953d639..b7fe150c8b90b09826f66feb8a158abf21793f2a 100644
--- a/.docs/system-services-metadata.md
+++ b/.docs/system-services-metadata.md
@@ -71,6 +71,10 @@ execution to the raw data. Any stale queries (query that have been executed by u
periodically being deleted from the query store based on the `DELETE_STALE_QUERIES_RATE` environment variable (defaults
to 60 seconds).
+Executing SQL queries through the Query Endpoint must fulfill some restrictions:
+
+* The SQL query does not contain at semicolon `;`
+
### Semantics
The service provides metadata to the table columns in the [Metadata Database](../system-databases-metadata) from
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6950219ea5e072b382dee782c5f32356e0ea13bb..edce253766e65fa113ab011223c93288bb5a3ba8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -131,12 +131,13 @@ test-analyse-service:
script:
- "pip install pipenv"
- "pipenv install gunicorn && pipenv install --dev --system --deploy"
- - "cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_determine_stats.py --junitxml=report.xml && coverage html && coverage report > ./coverage.txt"
+ - cd ./dbrepo-analyse-service/ && coverage run -m pytest test/test_determine_dt.py test/test_determine_pk.py test/test_s3_client.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt
- "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'"
artifacts:
when: always
paths:
- ./dbrepo-analyse-service/report.xml
+ - ./dbrepo-analyse-service/coverage.txt
expire_in: 1 days
reports:
junit: ./dbrepo-analyse-service/report.xml
diff --git a/dbrepo-analyse-service/pytest.ini b/dbrepo-analyse-service/pytest.ini
index 0aa3d82bb84628e89c95d0c4611688d344d2b148..21cb1c450f4c85de4ba416e0fe68ec9a93345aee 100644
--- a/dbrepo-analyse-service/pytest.ini
+++ b/dbrepo-analyse-service/pytest.ini
@@ -1,4 +1,3 @@
[pytest]
log_cli = 1
-log_level = info
-log_disable = main
\ No newline at end of file
+log_level = info
\ No newline at end of file
diff --git a/dbrepo-metadata-db/setup-schema.sql b/dbrepo-metadata-db/setup-schema.sql
index 047ecfe77783440def1475fa227a2fc09956cf2b..b267724b05bb77f8079e7bf65f2dcf1995266455 100644
--- a/dbrepo-metadata-db/setup-schema.sql
+++ b/dbrepo-metadata-db/setup-schema.sql
@@ -1,550 +1,553 @@
- BEGIN;
-
- CREATE TABLE IF NOT EXISTS `mdb_users`
- (
- id character varying(36) NOT NULL,
- username character varying(255) NOT NULL,
- firstname character varying(255),
- lastname character varying(255),
- email character varying(255) NOT NULL,
- orcid character varying(255),
- affiliation character varying(255),
- mariadb_password character varying(255) NOT NULL,
- theme_dark boolean,
- PRIMARY KEY (id),
- UNIQUE (username),
- UNIQUE (email)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_images`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- name character varying(255) NOT NULL,
- version character varying(255) NOT NULL,
- default_port integer NOT NULL,
- dialect character varying(255) NOT NULL,
- driver_class character varying(255) NOT NULL,
- jdbc_method character varying(255) NOT NULL,
- created timestamp NOT NULL DEFAULT NOW(),
- last_modified timestamp,
- PRIMARY KEY (id),
- UNIQUE (name, version)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_images_date`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- iid bigint NOT NULL,
- database_format character varying(255) NOT NULL,
- unix_format character varying(255) NOT NULL,
- example character varying(255) NOT NULL,
- has_time boolean NOT NULL,
- created_at timestamp NOT NULL DEFAULT NOW(),
- PRIMARY KEY (id),
- FOREIGN KEY (iid) REFERENCES mdb_images (id),
- UNIQUE (database_format, unix_format, example)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_containers`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- internal_name character varying(255) NOT NULL,
- name character varying(255) NOT NULL,
- host character varying(255) NOT NULL,
- port integer NOT NULL default 3306,
- ui_host character varying(255) NOT NULL default host,
- ui_port integer NOT NULL default port,
- ui_additional_flags text,
- sidecar_host character varying(255) NOT NULL,
- sidecar_port integer NOT NULL default 3305,
- image_id bigint NOT NULL,
- created timestamp NOT NULL DEFAULT NOW(),
- last_modified timestamp,
- privileged_username character varying(255) NOT NULL,
- privileged_password character varying(255) NOT NULL,
- PRIMARY KEY (id),
- FOREIGN KEY (image_id) REFERENCES mdb_images (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_data`
- (
- ID bigint NOT NULL AUTO_INCREMENT,
- PROVENANCE text,
- FileEncoding text,
- FileType character varying(100),
- Version text,
- Seperator text,
- PRIMARY KEY (ID)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_licenses`
- (
- identifier character varying(255) NOT NULL,
- uri text NOT NULL,
- PRIMARY KEY (identifier),
- UNIQUE (uri(200))
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_databases`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- cid bigint NOT NULL,
- name character varying(255) NOT NULL,
- internal_name character varying(255) NOT NULL,
- exchange_name character varying(255) NOT NULL,
- description text,
- engine character varying(20),
- is_public boolean NOT NULL DEFAULT TRUE,
- image longblob,
- created_by character varying(36),
- owned_by character varying(36),
- contact_person character varying(36),
- created timestamp NOT NULL DEFAULT NOW(),
- last_modified timestamp,
- PRIMARY KEY (id),
- FOREIGN KEY (cid) REFERENCES mdb_containers (id) /* currently we only support one-to-one */,
- FOREIGN KEY (created_by) REFERENCES mdb_users (id),
- FOREIGN KEY (owned_by) REFERENCES mdb_users (id),
- FOREIGN KEY (contact_person) REFERENCES mdb_users (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_databases_subjects`
- (
- dbid BIGINT NOT NULL,
- subjects character varying(255) NOT NULL,
- PRIMARY KEY (dbid, subjects)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_tables`
- (
- ID bigint NOT NULL AUTO_INCREMENT,
- tDBID bigint NOT NULL,
- internal_name character varying(255) NOT NULL,
- queue_name character varying(255) NOT NULL,
- routing_key character varying(255) NOT NULL,
- tName VARCHAR(50),
- tDescription TEXT,
- num_rows BIGINT,
- data_length BIGINT,
- max_data_length BIGINT,
- avg_row_length BIGINT,
- `separator` CHAR(1),
- quote CHAR(1),
- element_null VARCHAR(50),
- skip_lines BIGINT,
- element_true VARCHAR(50),
- element_false VARCHAR(50),
- Version TEXT,
- created timestamp NOT NULL DEFAULT NOW(),
- versioned boolean not null default true,
- created_by character varying(36) NOT NULL,
- owned_by character varying(36) NOT NULL,
- last_modified timestamp,
- PRIMARY KEY (ID),
- FOREIGN KEY (tDBID) REFERENCES mdb_databases (id),
- FOREIGN KEY (created_by) REFERENCES mdb_users (id),
- FOREIGN KEY (owned_by) REFERENCES mdb_users (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_columns`
- (
- ID BIGINT NOT NULL AUTO_INCREMENT,
- tID BIGINT NOT NULL,
- dfID BIGINT,
- cName VARCHAR(100),
- internal_name VARCHAR(100) NOT NULL,
- Datatype ENUM ('CHAR','VARCHAR','BINARY','VARBINARY','TINYBLOB','TINYTEXT','TEXT','BLOB','MEDIUMTEXT','MEDIUMBLOB','LONGTEXT','LONGBLOB','ENUM','SET','BIT','TINYINT','BOOL','SMALLINT','MEDIUMINT','INT','BIGINT','FLOAT','DOUBLE','DECIMAL','DATE','DATETIME','TIMESTAMP','TIME','YEAR'),
- length BIGINT NULL,
- ordinal_position INTEGER NOT NULL,
- is_primary_key BOOLEAN NOT NULL,
- index_length BIGINT NULL,
- size BIGINT,
- d BIGINT,
- auto_generated BOOLEAN DEFAULT false,
- is_null_allowed BOOLEAN NOT NULL DEFAULT true,
- val_min NUMERIC NULL,
- val_max NUMERIC NULL,
- mean NUMERIC NULL,
- median NUMERIC NULL,
- std_dev Numeric NULL,
- created timestamp NOT NULL DEFAULT NOW(),
- last_modified timestamp,
- FOREIGN KEY (tID) REFERENCES mdb_tables (ID) ON DELETE CASCADE,
- PRIMARY KEY (ID)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_columns_enums`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- column_id bigint NOT NULL,
- value CHARACTER VARYING(255) NOT NULL,
- FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE,
- PRIMARY KEY (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_columns_sets`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- column_id bigint NOT NULL,
- value CHARACTER VARYING(255) NOT NULL,
- FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE,
- PRIMARY KEY (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_columns_nom`
- (
- tID bigint,
- cID bigint,
- maxlength INTEGER,
- last_modified timestamp,
- created timestamp NOT NULL DEFAULT NOW(),
- FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID),
- PRIMARY KEY (tID, cID)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_columns_cat`
- (
- tID bigint,
- cID bigint,
- num_cat INTEGER,
- -- cat_array TEXT[],
- last_modified timestamp,
- created timestamp NOT NULL DEFAULT NOW(),
- FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID),
- PRIMARY KEY (tID, cID)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key`
- (
- fkid BIGINT NOT NULL AUTO_INCREMENT,
- tid BIGINT NOT NULL,
- rtid BIGINT NOT NULL,
- on_update VARCHAR(50) NULL,
- on_delete VARCHAR(50) NULL,
- position INT NULL,
- PRIMARY KEY (fkid),
- FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE,
- FOREIGN KEY (rtid) REFERENCES mdb_tables (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key_reference`
- (
- id BIGINT NOT NULL AUTO_INCREMENT,
- fkid BIGINT NOT NULL,
- cid BIGINT NOT NULL,
- rcid BIGINT NOT NULL,
- PRIMARY KEY (id),
- FOREIGN KEY (fkid) REFERENCES mdb_constraints_foreign_key (fkid) ON UPDATE CASCADE,
- FOREIGN KEY (cid) REFERENCES mdb_columns (id),
- FOREIGN KEY (rcid) REFERENCES mdb_columns (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_constraints_unique`
- (
- uid BIGINT NOT NULL AUTO_INCREMENT,
- tid BIGINT NOT NULL,
- position INT NULL,
- PRIMARY KEY (uid),
- FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE
- );
-
- CREATE TABLE IF NOT EXISTS `mdb_constraints_unique_columns`
- (
- id BIGINT NOT NULL AUTO_INCREMENT,
- uid BIGINT NOT NULL,
- cid BIGINT NOT NULL,
- PRIMARY KEY (id),
- FOREIGN KEY (uid) REFERENCES mdb_constraints_unique (uid),
- FOREIGN KEY (cid) REFERENCES mdb_columns (id) ON DELETE CASCADE
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_constraints_checks`
- (
- id BIGINT NOT NULL AUTO_INCREMENT,
- tid BIGINT NOT NULL,
- checks VARCHAR(255) NOT NULL,
- PRIMARY KEY (id),
- FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_concepts`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- uri text not null,
- name VARCHAR(255) null,
- description TEXT null,
- created timestamp NOT NULL DEFAULT NOW(),
- PRIMARY KEY (id),
- UNIQUE (uri(200))
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_units`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- uri text not null,
- name VARCHAR(255) null,
- description TEXT null,
- created timestamp NOT NULL DEFAULT NOW(),
- PRIMARY KEY (id),
- UNIQUE (uri(200))
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_columns_concepts`
- (
- id bigint NOT NULL,
- cID bigint NOT NULL,
- created timestamp NOT NULL DEFAULT NOW(),
- PRIMARY KEY (id, cid),
- FOREIGN KEY (cID) REFERENCES mdb_columns (ID)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_columns_units`
- (
- id bigint NOT NULL,
- cID bigint NOT NULL,
- created timestamp NOT NULL DEFAULT NOW(),
- PRIMARY KEY (id, cID),
- FOREIGN KEY (cID) REFERENCES mdb_columns (ID)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_view`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- vdbid bigint NOT NULL,
- vName VARCHAR(255) NOT NULL,
- internal_name VARCHAR(255) NOT NULL,
- Query TEXT NOT NULL,
- query_hash VARCHAR(255) NOT NULL,
- Public BOOLEAN NOT NULL,
- InitialView BOOLEAN NOT NULL,
- created timestamp NOT NULL DEFAULT NOW(),
- last_modified timestamp,
- created_by character varying(36) NOT NULL,
- PRIMARY KEY (id),
- FOREIGN KEY (vdbid) REFERENCES mdb_databases (id),
- FOREIGN KEY (created_by) REFERENCES mdb_users (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_banner_messages`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- type ENUM ('ERROR', 'WARNING', 'INFO') NOT NULL default 'INFO',
- message TEXT NOT NULL,
- link TEXT NULL,
- link_text VARCHAR(255) NULL,
- display_start timestamp NULL,
- display_end timestamp NULL,
- PRIMARY KEY (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_ontologies`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- prefix VARCHAR(8) NOT NULL,
- uri TEXT NOT NULL,
- uri_pattern TEXT,
- sparql_endpoint TEXT NULL,
- rdf_path TEXT NULL,
- last_modified timestamp,
- created timestamp NOT NULL DEFAULT NOW(),
- UNIQUE (prefix),
- UNIQUE (uri(200)),
- PRIMARY KEY (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_view_columns`
- (
- id BIGINT NOT NULL AUTO_INCREMENT,
- cid BIGINT NOT NULL,
- vid BIGINT NOT NULL,
- alias VARCHAR(100),
- ordinal_position INTEGER,
- PRIMARY KEY (id),
- FOREIGN KEY (vid) REFERENCES mdb_view (id),
- FOREIGN KEY (cid) REFERENCES mdb_columns (ID)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_identifiers`
- (
- id BIGINT NOT NULL AUTO_INCREMENT,
- dbid BIGINT,
- qid BIGINT,
- vid BIGINT,
- tid BIGINT,
- publisher VARCHAR(255) NOT NULL,
- language VARCHAR(2),
- publication_year INTEGER NOT NULL,
- publication_month INTEGER,
- publication_day INTEGER,
- identifier_type ENUM ('DATABASE', 'SUBSET', 'VIEW', 'TABLE') NOT NULL,
- query TEXT,
- query_normalized TEXT,
- query_hash VARCHAR(255),
- execution TIMESTAMP,
- result_hash VARCHAR(255),
- result_number BIGINT,
- doi VARCHAR(255),
- created TIMESTAMP NOT NULL DEFAULT NOW(),
- created_by VARCHAR(36) NOT NULL,
- last_modified TIMESTAMP,
- PRIMARY KEY (id), /* must be a single id from persistent identifier concept */
- FOREIGN KEY (dbid) REFERENCES mdb_databases (id),
- FOREIGN KEY (created_by) REFERENCES mdb_users (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_identifier_licenses`
- (
- pid bigint NOT NULL,
- license_id VARCHAR(255) NOT NULL,
- PRIMARY KEY (pid, license_id),
- FOREIGN KEY (pid) REFERENCES mdb_identifiers (id),
- FOREIGN KEY (license_id) REFERENCES mdb_licenses (identifier)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_identifier_titles`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- pid bigint NOT NULL,
- title text NOT NULL,
- title_type ENUM ('ALTERNATIVE_TITLE', 'SUBTITLE', 'TRANSLATED_TITLE', 'OTHER'),
- language VARCHAR(2),
- PRIMARY KEY (id),
- FOREIGN KEY (pid) REFERENCES mdb_identifiers (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_identifier_funders`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- pid bigint NOT NULL,
- funder_name VARCHAR(255) NOT NULL,
- funder_identifier TEXT,
- funder_identifier_type ENUM ('CROSSREF_FUNDER_ID', 'GRID', 'ISNI', 'ROR', 'OTHER'),
- scheme_uri text,
- award_number VARCHAR(255),
- award_title text,
- language VARCHAR(255),
- PRIMARY KEY (id),
- FOREIGN KEY (pid) REFERENCES mdb_identifiers (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_identifier_descriptions`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- pid bigint NOT NULL,
- description text NOT NULL,
- description_type ENUM ('ABSTRACT', 'METHODS', 'SERIES_INFORMATION', 'TABLE_OF_CONTENTS', 'TECHNICAL_INFO', 'OTHER'),
- language VARCHAR(2),
- PRIMARY KEY (id),
- FOREIGN KEY (pid) REFERENCES mdb_identifiers (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_related_identifiers`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- pid bigint NOT NULL,
- value varchar(255) NOT NULL,
- type varchar(255),
- relation varchar(255),
- PRIMARY KEY (id), /* must be a single id from persistent identifier concept */
- FOREIGN KEY (pid) REFERENCES mdb_identifiers (id),
- UNIQUE (pid, value)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_identifier_creators`
- (
- id bigint NOT NULL AUTO_INCREMENT,
- pid bigint NOT NULL,
- given_names text,
- family_name text,
- creator_name VARCHAR(255) NOT NULL,
- name_type ENUM ('PERSONAL', 'ORGANIZATIONAL') default 'PERSONAL',
- name_identifier text,
- name_identifier_scheme ENUM ('ROR', 'GRID', 'ISNI', 'ORCID'),
- name_identifier_scheme_uri text,
- affiliation VARCHAR(255),
- affiliation_identifier text,
- affiliation_identifier_scheme ENUM ('ROR', 'GRID', 'ISNI'),
- affiliation_identifier_scheme_uri text,
- PRIMARY KEY (id),
- FOREIGN KEY (pid) REFERENCES mdb_identifiers (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_feed`
- (
- fDBID bigint,
- fID bigint,
- fUserId character varying(36) not null,
- fDataID bigint REFERENCES mdb_data (ID),
- created timestamp NOT NULL DEFAULT NOW(),
- PRIMARY KEY (fDBID, fID, fUserId, fDataID),
- FOREIGN KEY (fDBID, fID) REFERENCES mdb_tables (tDBID, ID),
- FOREIGN KEY (fUserId) REFERENCES mdb_users (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_update`
- (
- uUserID character varying(255) NOT NULL,
- uDBID bigint NOT NULL,
- created timestamp NOT NULL DEFAULT NOW(),
- PRIMARY KEY (uUserID, uDBID),
- FOREIGN KEY (uDBID) REFERENCES mdb_databases (id)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_access`
- (
- aUserID character varying(255) NOT NULL,
- aDBID bigint REFERENCES mdb_databases (id),
- attime TIMESTAMP,
- download BOOLEAN,
- created timestamp NOT NULL DEFAULT NOW(),
- PRIMARY KEY (aUserID, aDBID)
- ) WITH SYSTEM VERSIONING;
-
- CREATE TABLE IF NOT EXISTS `mdb_have_access`
- (
- user_id character varying(36) NOT NULL,
- database_id bigint REFERENCES mdb_databases (id),
- access_type ENUM ('READ', 'WRITE_OWN', 'WRITE_ALL') NOT NULL,
- created timestamp NOT NULL DEFAULT NOW(),
- PRIMARY KEY (user_id, database_id),
- FOREIGN KEY (user_id) REFERENCES mdb_users (id)
- ) WITH SYSTEM VERSIONING;
-
- COMMIT;
- BEGIN;
-
- INSERT INTO `mdb_licenses` (identifier, uri)
- VALUES ('MIT', 'https://opensource.org/licenses/MIT'),
- ('GPL-3.0-only', 'https://www.gnu.org/licenses/gpl-3.0-standalone.html'),
- ('BSD-3-Clause', 'https://opensource.org/licenses/BSD-3-Clause'),
- ('BSD-4-Clause', 'http://directory.fsf.org/wiki/License:BSD_4Clause'),
- ('Apache-2.0', 'https://opensource.org/licenses/Apache-2.0'),
- ('CC0-1.0', 'https://creativecommons.org/publicdomain/zero/1.0/legalcode'),
- ('CC-BY-4.0', 'https://creativecommons.org/licenses/by/4.0/legalcode');
-
- INSERT INTO `mdb_images` (name, version, default_port, dialect, driver_class, jdbc_method)
- VALUES ('mariadb', '11.1.3', 3306, 'org.hibernate.dialect.MariaDBDialect', 'org.mariadb.jdbc.Driver', 'mariadb');
-
- INSERT INTO `mdb_images_date` (iid, database_format, unix_format, example, has_time)
- VALUES (1, '%Y-%c-%d %H:%i:%S.%f', 'yyyy-MM-dd HH:mm:ss.SSSSSS', '2022-01-30 13:44:25.499', true),
- (1, '%Y-%c-%d %H:%i:%S', 'yyyy-MM-dd HH:mm:ss', '2022-01-30 13:44:25', true),
- (1, '%Y-%c-%d', 'yyyy-MM-dd', '2022-01-30', false),
- (1, '%H:%i:%S', 'HH:mm:ss', '13:44:25', true);
-
- INSERT INTO `mdb_ontologies` (prefix, uri, uri_pattern, sparql_endpoint, rdf_path)
- VALUES ('om', 'http://www.ontology-of-units-of-measure.org/resource/om-2/',
- 'http://www.ontology-of-units-of-measure.org/resource/om-2/.*', null, 'rdf/om-2.0.rdf'),
- ('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql',
- null),
- ('mo', 'http://purl.org/ontology/mo/', 'http://purl.org/ontology/mo/.*', null, null),
- ('dc', 'http://purl.org/dc/elements/1.1/', null, null, null),
- ('xsd', 'http://www.w3.org/2001/XMLSchema#', null, null, null),
- ('tl', 'http://purl.org/NET/c4dm/timeline.owl#', null, null, null),
- ('foaf', 'http://xmlns.com/foaf/0.1/', null, null, null),
- ('schema', 'http://schema.org/', null, null, null),
- ('rdf', 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', null, null, null),
- ('rdfs', 'http://www.w3.org/2000/01/rdf-schema#', null, null, null),
- ('owl', 'http://www.w3.org/2002/07/owl#', null, null, null),
- ('prov', 'http://www.w3.org/ns/prov#', null, null, null),
- ('db', 'http://dbpedia.org', 'http://dbpedia.org/ontology/.*', 'http://dbpedia.org/sparql', null);
- COMMIT;
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS `mdb_users`
+(
+ id character varying(36) NOT NULL,
+ username character varying(255) NOT NULL,
+ firstname character varying(255),
+ lastname character varying(255),
+ email character varying(255) NOT NULL,
+ orcid character varying(255),
+ affiliation character varying(255),
+ mariadb_password character varying(255) NOT NULL,
+ theme_dark boolean,
+ PRIMARY KEY (id),
+ UNIQUE (username),
+ UNIQUE (email)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_images`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ name character varying(255) NOT NULL,
+ version character varying(255) NOT NULL,
+ default_port integer NOT NULL,
+ dialect character varying(255) NOT NULL,
+ driver_class character varying(255) NOT NULL,
+ jdbc_method character varying(255) NOT NULL,
+ created timestamp NOT NULL DEFAULT NOW(),
+ last_modified timestamp,
+ PRIMARY KEY (id),
+ UNIQUE (name, version)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_images_date`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ iid bigint NOT NULL,
+ database_format character varying(255) NOT NULL,
+ unix_format character varying(255) NOT NULL,
+ example character varying(255) NOT NULL,
+ has_time boolean NOT NULL,
+ created_at timestamp NOT NULL DEFAULT NOW(),
+ PRIMARY KEY (id),
+ FOREIGN KEY (iid) REFERENCES mdb_images (id),
+ UNIQUE (database_format, unix_format, example)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_containers`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ internal_name character varying(255) NOT NULL,
+ name character varying(255) NOT NULL,
+ host character varying(255) NOT NULL,
+ port integer NOT NULL default 3306,
+ ui_host character varying(255) NOT NULL default host,
+ ui_port integer NOT NULL default port,
+ ui_additional_flags text,
+ sidecar_host character varying(255) NOT NULL,
+ sidecar_port integer NOT NULL default 3305,
+ image_id bigint NOT NULL,
+ created timestamp NOT NULL DEFAULT NOW(),
+ last_modified timestamp,
+ privileged_username character varying(255) NOT NULL,
+ privileged_password character varying(255) NOT NULL,
+ PRIMARY KEY (id),
+ FOREIGN KEY (image_id) REFERENCES mdb_images (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_data`
+(
+ ID bigint NOT NULL AUTO_INCREMENT,
+ PROVENANCE text,
+ FileEncoding text,
+ FileType character varying(100),
+ Version text,
+ Seperator text,
+ PRIMARY KEY (ID)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_licenses`
+(
+ identifier character varying(255) NOT NULL,
+ uri text NOT NULL,
+ PRIMARY KEY (identifier),
+ UNIQUE (uri(200))
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_databases`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ cid bigint NOT NULL,
+ name character varying(255) NOT NULL,
+ internal_name character varying(255) NOT NULL,
+ exchange_name character varying(255) NOT NULL,
+ description text,
+ engine character varying(20),
+ is_public boolean NOT NULL DEFAULT TRUE,
+ image longblob,
+ created_by character varying(36),
+ owned_by character varying(36),
+ contact_person character varying(36),
+ created timestamp NOT NULL DEFAULT NOW(),
+ last_modified timestamp,
+ PRIMARY KEY (id),
+ FOREIGN KEY (cid) REFERENCES mdb_containers (id) /* currently we only support one-to-one */,
+ FOREIGN KEY (created_by) REFERENCES mdb_users (id),
+ FOREIGN KEY (owned_by) REFERENCES mdb_users (id),
+ FOREIGN KEY (contact_person) REFERENCES mdb_users (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_databases_subjects`
+(
+ dbid BIGINT NOT NULL,
+ subjects character varying(255) NOT NULL,
+ PRIMARY KEY (dbid, subjects)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_tables`
+(
+ ID bigint NOT NULL AUTO_INCREMENT,
+ tDBID bigint NOT NULL,
+ internal_name character varying(255) NOT NULL,
+ queue_name character varying(255) NOT NULL,
+ routing_key character varying(255) NOT NULL,
+ tName VARCHAR(50),
+ tDescription TEXT,
+ num_rows BIGINT,
+ data_length BIGINT,
+ max_data_length BIGINT,
+ avg_row_length BIGINT,
+ `separator` CHAR(1),
+ quote CHAR(1),
+ element_null VARCHAR(50),
+ skip_lines BIGINT,
+ element_true VARCHAR(50),
+ element_false VARCHAR(50),
+ Version TEXT,
+ created timestamp NOT NULL DEFAULT NOW(),
+ versioned boolean not null default true,
+ created_by character varying(36) NOT NULL,
+ owned_by character varying(36) NOT NULL,
+ processed_constraints BOOLEAN NOT NULL DEFAULT false,
+ last_modified timestamp,
+ PRIMARY KEY (ID),
+ FOREIGN KEY (tDBID) REFERENCES mdb_databases (id),
+ FOREIGN KEY (created_by) REFERENCES mdb_users (id),
+ FOREIGN KEY (owned_by) REFERENCES mdb_users (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_columns`
+(
+ ID BIGINT NOT NULL AUTO_INCREMENT,
+ tID BIGINT NOT NULL,
+ dfID BIGINT,
+ cName VARCHAR(100),
+ internal_name VARCHAR(100) NOT NULL,
+ Datatype ENUM ('CHAR','VARCHAR','BINARY','VARBINARY','TINYBLOB','TINYTEXT','TEXT','BLOB','MEDIUMTEXT','MEDIUMBLOB','LONGTEXT','LONGBLOB','ENUM','SET','BIT','TINYINT','BOOL','SMALLINT','MEDIUMINT','INT','BIGINT','FLOAT','DOUBLE','DECIMAL','DATE','DATETIME','TIMESTAMP','TIME','YEAR'),
+ length BIGINT NULL,
+ ordinal_position INTEGER NOT NULL,
+ is_primary_key BOOLEAN NOT NULL,
+ index_length BIGINT NULL,
+ size BIGINT,
+ d BIGINT,
+ auto_generated BOOLEAN DEFAULT false,
+ is_null_allowed BOOLEAN NOT NULL DEFAULT true,
+ val_min NUMERIC NULL,
+ val_max NUMERIC NULL,
+ mean NUMERIC NULL,
+ median NUMERIC NULL,
+ std_dev Numeric NULL,
+ created timestamp NOT NULL DEFAULT NOW(),
+ last_modified timestamp,
+ FOREIGN KEY (tID) REFERENCES mdb_tables (ID) ON DELETE CASCADE,
+ PRIMARY KEY (ID)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_columns_enums`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ column_id bigint NOT NULL,
+ value CHARACTER VARYING(255) NOT NULL,
+ FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE,
+ PRIMARY KEY (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_columns_sets`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ column_id bigint NOT NULL,
+ value CHARACTER VARYING(255) NOT NULL,
+ FOREIGN KEY (column_id) REFERENCES mdb_columns (ID) ON DELETE CASCADE,
+ PRIMARY KEY (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_columns_nom`
+(
+ tID bigint,
+ cID bigint,
+ maxlength INTEGER,
+ last_modified timestamp,
+ created timestamp NOT NULL DEFAULT NOW(),
+ FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID),
+ PRIMARY KEY (tID, cID)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_columns_cat`
+(
+ tID bigint,
+ cID bigint,
+ num_cat INTEGER,
+ -- cat_array TEXT[],
+ last_modified timestamp,
+ created timestamp NOT NULL DEFAULT NOW(),
+ FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID),
+ PRIMARY KEY (tID, cID)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key`
+(
+ fkid BIGINT NOT NULL AUTO_INCREMENT,
+ tid BIGINT NOT NULL,
+ rtid BIGINT NOT NULL,
+ name VARCHAR(255) NOT NULL,
+ on_update VARCHAR(50) NULL,
+ on_delete VARCHAR(50) NULL,
+ position INT NULL,
+ PRIMARY KEY (fkid),
+ FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE,
+ FOREIGN KEY (rtid) REFERENCES mdb_tables (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_constraints_foreign_key_reference`
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ fkid BIGINT NOT NULL,
+ cid BIGINT NOT NULL,
+ rcid BIGINT NOT NULL,
+ PRIMARY KEY (id),
+ FOREIGN KEY (fkid) REFERENCES mdb_constraints_foreign_key (fkid) ON UPDATE CASCADE,
+ FOREIGN KEY (cid) REFERENCES mdb_columns (id),
+ FOREIGN KEY (rcid) REFERENCES mdb_columns (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_constraints_unique`
+(
+ uid BIGINT NOT NULL AUTO_INCREMENT,
+ name VARCHAR(255) NOT NULL,
+ tid BIGINT NOT NULL,
+ position INT NULL,
+ PRIMARY KEY (uid),
+ FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS `mdb_constraints_unique_columns`
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ uid BIGINT NOT NULL,
+ cid BIGINT NOT NULL,
+ PRIMARY KEY (id),
+ FOREIGN KEY (uid) REFERENCES mdb_constraints_unique (uid),
+ FOREIGN KEY (cid) REFERENCES mdb_columns (id) ON DELETE CASCADE
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_constraints_checks`
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ tid BIGINT NOT NULL,
+ checks VARCHAR(255) NOT NULL,
+ PRIMARY KEY (id),
+ FOREIGN KEY (tid) REFERENCES mdb_tables (id) ON DELETE CASCADE
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_concepts`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ uri text not null,
+ name VARCHAR(255) null,
+ description TEXT null,
+ created timestamp NOT NULL DEFAULT NOW(),
+ PRIMARY KEY (id),
+ UNIQUE (uri(200))
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_units`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ uri text not null,
+ name VARCHAR(255) null,
+ description TEXT null,
+ created timestamp NOT NULL DEFAULT NOW(),
+ PRIMARY KEY (id),
+ UNIQUE (uri(200))
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_columns_concepts`
+(
+ id bigint NOT NULL,
+ cID bigint NOT NULL,
+ created timestamp NOT NULL DEFAULT NOW(),
+ PRIMARY KEY (id, cid),
+ FOREIGN KEY (cID) REFERENCES mdb_columns (ID)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_columns_units`
+(
+ id bigint NOT NULL,
+ cID bigint NOT NULL,
+ created timestamp NOT NULL DEFAULT NOW(),
+ PRIMARY KEY (id, cID),
+ FOREIGN KEY (cID) REFERENCES mdb_columns (ID)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_view`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ vdbid bigint NOT NULL,
+ vName VARCHAR(255) NOT NULL,
+ internal_name VARCHAR(255) NOT NULL,
+ Query TEXT NOT NULL,
+ query_hash VARCHAR(255) NOT NULL,
+ Public BOOLEAN NOT NULL,
+ InitialView BOOLEAN NOT NULL,
+ created timestamp NOT NULL DEFAULT NOW(),
+ last_modified timestamp,
+ created_by character varying(36) NOT NULL,
+ PRIMARY KEY (id),
+ FOREIGN KEY (vdbid) REFERENCES mdb_databases (id),
+ FOREIGN KEY (created_by) REFERENCES mdb_users (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_banner_messages`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ type ENUM ('ERROR', 'WARNING', 'INFO') NOT NULL default 'INFO',
+ message TEXT NOT NULL,
+ link TEXT NULL,
+ link_text VARCHAR(255) NULL,
+ display_start timestamp NULL,
+ display_end timestamp NULL,
+ PRIMARY KEY (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_ontologies`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ prefix VARCHAR(8) NOT NULL,
+ uri TEXT NOT NULL,
+ uri_pattern TEXT,
+ sparql_endpoint TEXT NULL,
+ rdf_path TEXT NULL,
+ last_modified timestamp,
+ created timestamp NOT NULL DEFAULT NOW(),
+ UNIQUE (prefix),
+ UNIQUE (uri(200)),
+ PRIMARY KEY (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_view_columns`
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ cid BIGINT NOT NULL,
+ vid BIGINT NOT NULL,
+ alias VARCHAR(100),
+ ordinal_position INTEGER,
+ PRIMARY KEY (id),
+ FOREIGN KEY (vid) REFERENCES mdb_view (id),
+ FOREIGN KEY (cid) REFERENCES mdb_columns (ID)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_identifiers`
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ dbid BIGINT,
+ qid BIGINT,
+ vid BIGINT,
+ tid BIGINT,
+ publisher VARCHAR(255) NOT NULL,
+ language VARCHAR(2),
+ publication_year INTEGER NOT NULL,
+ publication_month INTEGER,
+ publication_day INTEGER,
+ identifier_type ENUM ('DATABASE', 'SUBSET', 'VIEW', 'TABLE') NOT NULL,
+ query TEXT,
+ query_normalized TEXT,
+ query_hash VARCHAR(255),
+ execution TIMESTAMP,
+ result_hash VARCHAR(255),
+ result_number BIGINT,
+ doi VARCHAR(255),
+ created TIMESTAMP NOT NULL DEFAULT NOW(),
+ created_by VARCHAR(36) NOT NULL,
+ last_modified TIMESTAMP,
+ PRIMARY KEY (id), /* must be a single id from persistent identifier concept */
+ FOREIGN KEY (dbid) REFERENCES mdb_databases (id),
+ FOREIGN KEY (created_by) REFERENCES mdb_users (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_identifier_licenses`
+(
+ pid bigint NOT NULL,
+ license_id VARCHAR(255) NOT NULL,
+ PRIMARY KEY (pid, license_id),
+ FOREIGN KEY (pid) REFERENCES mdb_identifiers (id),
+ FOREIGN KEY (license_id) REFERENCES mdb_licenses (identifier)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_identifier_titles`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ pid bigint NOT NULL,
+ title text NOT NULL,
+ title_type ENUM ('ALTERNATIVE_TITLE', 'SUBTITLE', 'TRANSLATED_TITLE', 'OTHER'),
+ language VARCHAR(2),
+ PRIMARY KEY (id),
+ FOREIGN KEY (pid) REFERENCES mdb_identifiers (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_identifier_funders`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ pid bigint NOT NULL,
+ funder_name VARCHAR(255) NOT NULL,
+ funder_identifier TEXT,
+ funder_identifier_type ENUM ('CROSSREF_FUNDER_ID', 'GRID', 'ISNI', 'ROR', 'OTHER'),
+ scheme_uri text,
+ award_number VARCHAR(255),
+ award_title text,
+ language VARCHAR(255),
+ PRIMARY KEY (id),
+ FOREIGN KEY (pid) REFERENCES mdb_identifiers (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_identifier_descriptions`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ pid bigint NOT NULL,
+ description text NOT NULL,
+ description_type ENUM ('ABSTRACT', 'METHODS', 'SERIES_INFORMATION', 'TABLE_OF_CONTENTS', 'TECHNICAL_INFO', 'OTHER'),
+ language VARCHAR(2),
+ PRIMARY KEY (id),
+ FOREIGN KEY (pid) REFERENCES mdb_identifiers (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_related_identifiers`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ pid bigint NOT NULL,
+ value varchar(255) NOT NULL,
+ type varchar(255),
+ relation varchar(255),
+ PRIMARY KEY (id), /* must be a single id from persistent identifier concept */
+ FOREIGN KEY (pid) REFERENCES mdb_identifiers (id),
+ UNIQUE (pid, value)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_identifier_creators`
+(
+ id bigint NOT NULL AUTO_INCREMENT,
+ pid bigint NOT NULL,
+ given_names text,
+ family_name text,
+ creator_name VARCHAR(255) NOT NULL,
+ name_type ENUM ('PERSONAL', 'ORGANIZATIONAL') default 'PERSONAL',
+ name_identifier text,
+ name_identifier_scheme ENUM ('ROR', 'GRID', 'ISNI', 'ORCID'),
+ name_identifier_scheme_uri text,
+ affiliation VARCHAR(255),
+ affiliation_identifier text,
+ affiliation_identifier_scheme ENUM ('ROR', 'GRID', 'ISNI'),
+ affiliation_identifier_scheme_uri text,
+ PRIMARY KEY (id),
+ FOREIGN KEY (pid) REFERENCES mdb_identifiers (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_feed`
+(
+ fDBID bigint,
+ fID bigint,
+ fUserId character varying(36) not null,
+ fDataID bigint REFERENCES mdb_data (ID),
+ created timestamp NOT NULL DEFAULT NOW(),
+ PRIMARY KEY (fDBID, fID, fUserId, fDataID),
+ FOREIGN KEY (fDBID, fID) REFERENCES mdb_tables (tDBID, ID),
+ FOREIGN KEY (fUserId) REFERENCES mdb_users (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_update`
+(
+ uUserID character varying(255) NOT NULL,
+ uDBID bigint NOT NULL,
+ created timestamp NOT NULL DEFAULT NOW(),
+ PRIMARY KEY (uUserID, uDBID),
+ FOREIGN KEY (uDBID) REFERENCES mdb_databases (id)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_access`
+(
+ aUserID character varying(255) NOT NULL,
+ aDBID bigint REFERENCES mdb_databases (id),
+ attime TIMESTAMP,
+ download BOOLEAN,
+ created timestamp NOT NULL DEFAULT NOW(),
+ PRIMARY KEY (aUserID, aDBID)
+) WITH SYSTEM VERSIONING;
+
+CREATE TABLE IF NOT EXISTS `mdb_have_access`
+(
+ user_id character varying(36) NOT NULL,
+ database_id bigint REFERENCES mdb_databases (id),
+ access_type ENUM ('READ', 'WRITE_OWN', 'WRITE_ALL') NOT NULL,
+ created timestamp NOT NULL DEFAULT NOW(),
+ PRIMARY KEY (user_id, database_id),
+ FOREIGN KEY (user_id) REFERENCES mdb_users (id)
+) WITH SYSTEM VERSIONING;
+
+COMMIT;
+BEGIN;
+
+INSERT INTO `mdb_licenses` (identifier, uri)
+VALUES ('MIT', 'https://opensource.org/licenses/MIT'),
+ ('GPL-3.0-only', 'https://www.gnu.org/licenses/gpl-3.0-standalone.html'),
+ ('BSD-3-Clause', 'https://opensource.org/licenses/BSD-3-Clause'),
+ ('BSD-4-Clause', 'http://directory.fsf.org/wiki/License:BSD_4Clause'),
+ ('Apache-2.0', 'https://opensource.org/licenses/Apache-2.0'),
+ ('CC0-1.0', 'https://creativecommons.org/publicdomain/zero/1.0/legalcode'),
+ ('CC-BY-4.0', 'https://creativecommons.org/licenses/by/4.0/legalcode');
+
+INSERT INTO `mdb_images` (name, version, default_port, dialect, driver_class, jdbc_method)
+VALUES ('mariadb', '11.1.3', 3306, 'org.hibernate.dialect.MariaDBDialect', 'org.mariadb.jdbc.Driver', 'mariadb');
+
+INSERT INTO `mdb_images_date` (iid, database_format, unix_format, example, has_time)
+VALUES (1, '%Y-%c-%d %H:%i:%S.%f', 'yyyy-MM-dd HH:mm:ss.SSSSSS', '2022-01-30 13:44:25.499', true),
+ (1, '%Y-%c-%d %H:%i:%S', 'yyyy-MM-dd HH:mm:ss', '2022-01-30 13:44:25', true),
+ (1, '%Y-%c-%d', 'yyyy-MM-dd', '2022-01-30', false),
+ (1, '%H:%i:%S', 'HH:mm:ss', '13:44:25', true);
+
+INSERT INTO `mdb_ontologies` (prefix, uri, uri_pattern, sparql_endpoint, rdf_path)
+VALUES ('om', 'http://www.ontology-of-units-of-measure.org/resource/om-2/',
+ 'http://www.ontology-of-units-of-measure.org/resource/om-2/.*', null, 'rdf/om-2.0.rdf'),
+ ('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql',
+ null),
+ ('mo', 'http://purl.org/ontology/mo/', 'http://purl.org/ontology/mo/.*', null, null),
+ ('dc', 'http://purl.org/dc/elements/1.1/', null, null, null),
+ ('xsd', 'http://www.w3.org/2001/XMLSchema#', null, null, null),
+ ('tl', 'http://purl.org/NET/c4dm/timeline.owl#', null, null, null),
+ ('foaf', 'http://xmlns.com/foaf/0.1/', null, null, null),
+ ('schema', 'http://schema.org/', null, null, null),
+ ('rdf', 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', null, null, null),
+ ('rdfs', 'http://www.w3.org/2000/01/rdf-schema#', null, null, null),
+ ('owl', 'http://www.w3.org/2002/07/owl#', null, null, null),
+ ('prov', 'http://www.w3.org/ns/prov#', null, null, null),
+ ('db', 'http://dbpedia.org', 'http://dbpedia.org/ontology/.*', 'http://dbpedia.org/sparql', null);
+COMMIT;
diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java
index d94ed58767610d7300c593a1150b75aed9b175f1..b865f7892cf74fcf1ae4a0feeec3075c87fa5817 100644
--- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java
+++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/query/ImportDto.java
@@ -43,4 +43,8 @@ public class ImportDto {
@Schema(example = "\"")
private Character quote;
+
+ @JsonProperty("line_termination")
+ @Schema(example = "\\r\\n")
+ private String lineTermination;
}
diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/constraints/foreignKey/ForeignKeyDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/constraints/foreignKey/ForeignKeyDto.java
index bcc3c6d8d139aacb3e831a2e03a3025aff7bdd88..e577b033c46dd0c1d9cc030a018d1a6f8b572ac4 100644
--- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/constraints/foreignKey/ForeignKeyDto.java
+++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/constraints/foreignKey/ForeignKeyDto.java
@@ -19,6 +19,8 @@ import java.util.List;
@ToString
public class ForeignKeyDto {
+ private String name;
+
@org.springframework.data.annotation.Transient
private List<ColumnDto> columns;
diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java
index 10f13c3bd61c2eeac9c83fe2eff664eec4a99e46..a74bde27fe3afd1f4ae2d1878541efdf4759773a 100644
--- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java
+++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/ViewColumn.java
@@ -26,7 +26,7 @@ public class ViewColumn implements Comparable<ViewColumn> {
@Column(updatable = false, nullable = false)
private Long id;
- @Column(updatable = false, nullable = false)
+ @Column(updatable = false)
private String alias;
@Column(nullable = false)
diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/Table.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/Table.java
index 5ada0ba03bf50c5e844b9b69cc2a459b4dba0403..02ab0e59410b4b28c14c7dd0a86810aae4846ecf 100644
--- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/Table.java
+++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/Table.java
@@ -132,6 +132,9 @@ public class Table {
@Column(columnDefinition = "TIMESTAMP")
private Instant lastModified;
+ @Column(name = "processed_constraints", nullable = false)
+ private Boolean processedConstraints;
+
@Override
public boolean equals(Object o) {
if (o == this) {
diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/foreignKey/ForeignKey.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/foreignKey/ForeignKey.java
index 2ec4f2ba5d7913cdca0a14b298773571976bfec7..92050024e5c0d24dde314a915c4746dc315c99ef 100644
--- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/foreignKey/ForeignKey.java
+++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/foreignKey/ForeignKey.java
@@ -26,6 +26,9 @@ public class ForeignKey {
@Column(updatable = false, nullable = false)
private Long fkid;
+ @Column(updatable = false, nullable = false)
+ private String name;
+
@ToString.Exclude
@org.springframework.data.annotation.Transient
@ManyToOne(fetch = FetchType.LAZY, cascade = CascadeType.MERGE)
diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/unique/Unique.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/unique/Unique.java
index 8cdebed830bdc4bb940002db92d49a0a4262ead4..d373339f2d0ac4e3a70344a914882087de907119 100644
--- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/unique/Unique.java
+++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/constraints/unique/Unique.java
@@ -27,6 +27,9 @@ public class Unique {
@Column(updatable = false, nullable = false)
private Long uid;
+ @Column(updatable = false, nullable = false)
+ private String name;
+
@ToString.Exclude
@org.springframework.data.annotation.Transient
@ManyToOne(fetch = FetchType.LAZY, cascade = CascadeType.MERGE)
diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/DatabaseMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/DatabaseMapper.java
index 7bd1c5d0eb55eb4aa815d9662732d9547f38aef4..764d79cc29130f235f5896dff35de5967c3b8c61 100644
--- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/DatabaseMapper.java
+++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/DatabaseMapper.java
@@ -117,7 +117,7 @@ public interface DatabaseMapper {
default PreparedStatement databaseToDatabaseMetadata(Connection connection, Database database) throws QueryMalformedException {
final StringBuilder statement = new StringBuilder("SELECT t.`TABLE_NAME`, t.`TABLE_TYPE`, t.`TABLE_ROWS`, t.`AVG_ROW_LENGTH`, t.`DATA_LENGTH`, t.`MAX_DATA_LENGTH`, COALESCE(t.`CREATE_TIME`, NOW()) as `CREATE_TIME`, t.`UPDATE_TIME`, v.`VIEW_DEFINITION` FROM information_schema.TABLES t LEFT JOIN information_schema.VIEWS v ON t.`TABLE_NAME` = v.`TABLE_NAME` WHERE t.`TABLE_SCHEMA` = '")
.append(database.getInternalName())
- .append("' AND t.`TABLE_TYPE` IN ('BASE TABLE', 'SYSTEM VERSIONED', 'VIEW') AND t.`TABLE_NAME` NOT IN ('qs_queries', '_tmp') AND t.`TABLE_NAME` NOT LIKE 'hs_%' AND t.`TABLE_NAME` NOT LIKE '%_temporary'");
+ .append("' AND t.`TABLE_TYPE` IN ('BASE TABLE', 'SYSTEM VERSIONED', 'VIEW') AND t.`TABLE_NAME` != 'qs_queries' AND t.`TABLE_NAME` NOT LIKE 'hs_%'");
log.trace("statement={}", statement);
try {
return connection.prepareStatement(statement.toString());
diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java
index 31bb6c0881a69fe524e3b83ae164b279efcbfef4..536810dfd735d5b9c4f135c65119bab8779ffc21 100644
--- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java
+++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/QueryMapper.java
@@ -9,9 +9,12 @@ import at.tuwien.api.database.table.TableCsvDto;
import at.tuwien.api.database.table.TableHistoryDto;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.View;
+import at.tuwien.entities.database.ViewColumn;
import at.tuwien.entities.database.table.Table;
import at.tuwien.entities.database.table.columns.TableColumn;
import at.tuwien.entities.database.table.columns.TableColumnType;
+import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKey;
+import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKeyReference;
import at.tuwien.exception.ImageNotSupportedException;
import at.tuwien.exception.QueryMalformedException;
import at.tuwien.exception.QueryStoreException;
@@ -47,6 +50,7 @@ import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
@Mapper(componentModel = "spring", imports = {LinkedList.class})
public interface QueryMapper {
@@ -122,44 +126,13 @@ public interface QueryMapper {
.build();
}
- default void importCsvQuery(Connection connection, Table table, ImportDto csv) throws SQLException {
- final Statement statement = connection.createStatement();
- final StringBuilder query0 = new StringBuilder("CREATE TABLE IF NOT EXISTS `")
- .append(table.getDatabase().getInternalName())
- .append("`.`")
- .append(table.getInternalName())
- .append("_temporary`")
- .append(" LIKE `")
- .append(table.getDatabase().getInternalName())
- .append("`.`")
- .append(table.getInternalName())
- .append("`;");
- log.trace("mapped create temporary table statement: {}", query0);
- statement.execute(query0.toString());
- final String query1 = pathToRawInsertQuery(table, csv);
- log.trace("mapped import csv statement: {}", query1);
- statement.execute(query1.toString());
- final String query2 = generateInsertFromTemporaryTableSQL(table);
- log.trace("mapped import table statement: {}", query2);
- statement.execute(query2.toString());
- final StringBuilder query3 = new StringBuilder("DROP TABLE IF EXISTS `")
- .append(table.getDatabase().getInternalName())
- .append("`.`")
- .append(table.getInternalName())
- .append("_temporary`;");
- log.trace("mapped drop temporary table statement: {}", query3);
- statement.execute(query3.toString());
-
- }
-
- default String pathToRawInsertQuery(Table table, ImportDto data) {
+ default PreparedStatement pathToRawInsertQuery(Connection connection, Table table, ImportDto data) throws QueryMalformedException {
final StringBuilder statement = new StringBuilder("LOAD DATA INFILE '/tmp/")
.append(data.getLocation())
- .append("' INTO TABLE `")
+ .append("' REPLACE INTO TABLE `")
.append(table.getDatabase().getInternalName())
.append("`.`")
.append(table.getInternalName())
- .append("_temporary")
.append("` CHARACTER SET utf8 FIELDS TERMINATED BY '")
.append(data.getSeparator())
.append("'");
@@ -168,7 +141,10 @@ public interface QueryMapper {
.append(data.getQuote())
.append("'");
}
- statement.append(data.getSkipLines() != null ? (" IGNORE " + data.getSkipLines() + " LINES") : "")
+ statement.append(" LINES TERMINATED BY '")
+ .append(data.getLineTermination())
+ .append("'")
+ .append(data.getSkipLines() != null ? (" IGNORE " + data.getSkipLines() + " LINES") : "")
.append(" (");
final StringBuilder set = new StringBuilder();
int[] idx = new int[]{0};
@@ -200,7 +176,14 @@ public interface QueryMapper {
statement.append(")")
.append(set.length() != 0 ? (" SET " + set) : "")
.append(";");
- return statement.toString();
+ try {
+ final PreparedStatement pstmt = connection.prepareStatement(statement.toString());
+ log.trace("mapped import csv query {} to prepared statement {}", table.getName(), pstmt);
+ return pstmt;
+ } catch (SQLException e) {
+ log.error("Failed to prepare statement {}: {}", statement, e.getMessage());
+ throw new QueryMalformedException("Failed to prepare statement:" + e.getMessage(), e);
+ }
}
default void columnToBoolSet(ImportDto data, TableColumn column, StringBuilder set) {
@@ -209,18 +192,14 @@ public interface QueryMapper {
.append("`")
.append(column.getInternalName())
.append("` = ");
- if (data.getNullElement() != null) {
- log.trace("import has null element present");
- set.append("IF(!STRCMP(@")
- .append(column.getInternalName())
- .append(",'")
- .append(data.getNullElement())
- .append("'),NULL,");
- columnToBoolSet2(data, column, set);
- set.append(")");
- return;
- }
+ log.trace("import has null element present");
+ set.append("IF(!STRCMP(@")
+ .append(column.getInternalName())
+ .append(",'")
+ .append(data.getNullElement())
+ .append("'),NULL,");
columnToBoolSet2(data, column, set);
+ set.append(")");
}
default void columnToBoolSet2(ImportDto data, TableColumn column, StringBuilder set) {
@@ -285,19 +264,14 @@ public interface QueryMapper {
.append("`")
.append(column.getInternalName())
.append("` = ");
- if (data.getNullElement() != null) {
- log.trace("import has null element present");
- set.append("IF(STRCMP(@")
- .append(column.getInternalName())
- .append(",'")
- .append(data.getNullElement())
- .append("'), @")
- .append(column.getInternalName())
- .append(", NULL)");
- return;
- }
- set.append("@")
- .append(column.getInternalName());
+ log.trace("import has null element present");
+ set.append("IF(STRCMP(@")
+ .append(column.getInternalName())
+ .append(",'")
+ .append(data.getNullElement())
+ .append("'), @")
+ .append(column.getInternalName())
+ .append(", NULL)");
}
default void columnToDateSet(ImportDto data, TableColumn column, StringBuilder set) {
@@ -306,24 +280,14 @@ public interface QueryMapper {
.append("`")
.append(column.getInternalName())
.append("` = STR_TO_DATE(");
- if (data.getNullElement() != null) {
- log.trace("import has null element present");
- set.append("IF(STRCMP(@")
- .append(column.getInternalName())
- .append(",'")
- .append(data.getNullElement())
- .append("'), @")
- .append(column.getInternalName())
- .append(", NULL), '")
- .append(column.getDateFormat()
- .getDatabaseFormat()
- .replace('\'', '\\'))
- .append("')");
- return;
- }
- set.append("@")
+ log.trace("import has null element present");
+ set.append("IF(STRCMP(@")
.append(column.getInternalName())
- .append(", '")
+ .append(",'")
+ .append(data.getNullElement())
+ .append("'), @")
+ .append(column.getInternalName())
+ .append(", NULL), '")
.append(column.getDateFormat()
.getDatabaseFormat()
.replace('\'', '\\'))
@@ -500,18 +464,18 @@ public interface QueryMapper {
for (Map.Entry<String, Object> entry : data.getKeys().entrySet()) {
final Optional<TableColumn> optional = table.getColumns()
.stream()
- .filter(c -> c.getInternalName().equals(entry.getKey()))
+ .filter(c -> c.getInternalName().equals(entry.getKey().replace("`", "")))
.findFirst();
if (optional.isEmpty()) {
- log.error("Failed to find column with name {}, available names: {}", entry.getKey(), data.getKeys().keySet());
- throw new QueryMalformedException("Failed to find column");
+ log.error("Failed to find column with name {} in table {}", entry.getKey(), table.getInternalName());
+ throw new QueryMalformedException("Failed to find column with name " + entry.getKey() + " in table " + table.getInternalName());
}
prepareStatementWithColumnTypeObject(pstmt, optional.get().getColumnType(), i++, entry.getValue());
}
return pstmt;
} catch (SQLException e) {
- log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage());
- throw new QueryMalformedException("Failed to prepare statement", e);
+ log.error("Failed to prepare statement {}: {}", statement, e.getMessage());
+ throw new QueryMalformedException("Failed to prepare statement: " + e.getMessage(), e);
}
}
@@ -697,10 +661,21 @@ public interface QueryMapper {
}
}
}
- final List<TableColumn> allColumns = database.getTables()
- .stream()
- .map(Table::getColumns)
- .flatMap(List::stream)
+ final List<ViewColumn> allColumns = Stream.of(database.getViews()
+ .stream()
+ .map(View::getColumns)
+ .flatMap(List::stream),
+ database.getTables()
+ .stream()
+ .map(Table::getColumns)
+ .flatMap(List::stream)
+ .map(c -> ViewColumn.builder()
+ .column(c)
+ .alias(c.getAlias())
+ .ordinalPosition(c.getOrdinalPosition())
+ .build())
+ )
+ .flatMap(i -> i)
.toList();
log.trace("columns referenced in the from-clause and join-clause(s): {}", clauses);
/* Checking if all tables or views exist */
@@ -725,20 +700,24 @@ public interface QueryMapper {
log.error("Failed to find table/view {} (with designator {})", column.getTable().getName(), column.getTable().getAlias());
throw new JSQLParserException("Failed to find table/view " + column.getTable().getName() + " (with alias " + column.getTable().getAlias() + ")");
}
- final Optional<TableColumn> optionalColumn = allColumns.stream()
- .filter(c -> c.getInternalName().equals(column.getColumnName().replace("`", "")))
- .filter(c -> columnMatches(c, optional.get().getName().replace("`", "")))
+ final String columnName = column.getColumnName().replace("`", "");
+ final String tableOrView = optional.get().getName().replace("`", "");
+ final List<ViewColumn> filteredColumns = allColumns.stream()
+ .filter(c -> (c.getAlias() != null && c.getAlias().equals(columnName)) || c.getColumn().getInternalName().equals(columnName))
+ .toList();
+ final Optional<ViewColumn> optionalColumn = filteredColumns.stream()
+ .filter(c -> columnMatches(c, tableOrView))
.findFirst();
if (optionalColumn.isEmpty()) {
- log.error("Failed to find column with name {} in {}", column.getColumnName(), allColumns.stream().map(TableColumn::getInternalName).toList());
- throw new JSQLParserException("Failed to find column with name " + column.getColumnName() + " in " + allColumns.stream().map(TableColumn::getInternalName).toList());
+ log.error("Failed to find column with name {} of table/view {} in {}", columnName, tableOrView, filteredColumns.stream().map(c -> c.getColumn().getTable().getInternalName() + "." + c.getColumn().getInternalName()).toList());
+ throw new JSQLParserException("Failed to find column with name " + columnName + " of table/view " + tableOrView);
}
- final TableColumn aliasColumn = optionalColumn.get();
+ final ViewColumn resultColumn = optionalColumn.get();
if (item.getAlias() != null) {
- aliasColumn.setAlias(item.getAlias().getName().replace("`", ""));
+ resultColumn.getColumn().setAlias(item.getAlias().getName().replace("`", ""));
}
- log.trace("found column with internal name {} and alias {}", aliasColumn.getInternalName(), aliasColumn.getAlias());
- columns.add(aliasColumn);
+ log.trace("found column with internal name {} and alias {}", resultColumn.getColumn().getInternalName(), resultColumn.getAlias());
+ columns.add(resultColumn.getColumn());
}
return columns;
}
@@ -785,18 +764,28 @@ public interface QueryMapper {
}
@Transactional(readOnly = true)
- default boolean columnMatches(TableColumn column, String tableOrView) {
- if (column.getTable().getInternalName().equals(tableOrView)) {
- /* matches table name */
+ default boolean columnMatches(ViewColumn column, String tableOrView) {
+ if (column.getView() != null && column.getView().getInternalName().equals(tableOrView)) {
+ log.trace("view {} found in column table", tableOrView);
+ return true;
+ }
+ if (column.getColumn().getTable().getInternalName().equals(tableOrView)) {
+ log.trace("table {} found in column table", tableOrView);
return true;
}
- if (column.getViews() == null) {
+ if (column.getColumn().getViews() == null) {
+ log.trace("table/view {} not found among column views: empty list", tableOrView);
return false;
}
- /* maybe matches one of the views */
- return column.getViews()
+ /* maybe matches one of the other views */
+ final boolean found = column.getColumn()
+ .getViews()
.stream()
.anyMatch(v -> v.getInternalName().equals(tableOrView));
+ if (!found) {
+ log.trace("table/view {} not found among column views: {}", tableOrView, column.getColumn().getViews().stream().map(View::getInternalName).toList());
+ }
+ return found;
}
default PreparedStatement obtainTableMetadataRawQuery(Connection connection, String databaseName, String tableName) throws QueryMalformedException {
@@ -810,22 +799,31 @@ public interface QueryMapper {
return connection.prepareStatement(statement.toString());
} catch (SQLException e) {
log.error("Failed to prepare statement {}: {}", statement, e.getMessage());
- throw new QueryMalformedException("Failed to prepare statement", e);
+ throw new QueryMalformedException("Failed to prepare statement: " + e.getMessage(), e);
}
}
+ default ForeignKeyReference foreignKeyToForeignKeyReference(ForeignKey foreignKey, TableColumn column,
+ TableColumn referencedColumn) {
+ return ForeignKeyReference.builder()
+ .foreignKey(foreignKey)
+ .column(column)
+ .referencedColumn(referencedColumn)
+ .build();
+ }
+
default PreparedStatement databaseToDatabaseConstraintMetadata(Connection connection, String databaseName, String tableName) throws QueryMalformedException {
- final StringBuilder statement = new StringBuilder("SELECT tc.`CONSTRAINT_TYPE`, cc.`CONSTRAINT_NAME`, cc.`LEVEL`, cc.`CHECK_CLAUSE`, rc.`UNIQUE_CONSTRAINT_NAME`, rc.`REFERENCED_TABLE_NAME` FROM information_schema.`TABLE_CONSTRAINTS` tc LEFT JOIN information_schema.`CHECK_CONSTRAINTS` cc ON tc.`CONSTRAINT_SCHEMA` = cc.`CONSTRAINT_SCHEMA` AND tc.`TABLE_NAME` = cc.`TABLE_NAME` AND tc.`CONSTRAINT_TYPE` = 'CHECK' LEFT JOIN information_schema.`REFERENTIAL_CONSTRAINTS` rc ON tc.`CONSTRAINT_SCHEMA` = rc.`CONSTRAINT_SCHEMA` AND tc.`TABLE_NAME` = rc.`TABLE_NAME` AND tc.`CONSTRAINT_TYPE` = 'FOREIGN KEY' WHERE tc.`TABLE_SCHEMA` = '")
+ final StringBuilder statement = new StringBuilder("SELECT tc.`CONSTRAINT_TYPE`, tc.`CONSTRAINT_NAME`, cc.`LEVEL`, cc.`CHECK_CLAUSE`, rc.`UNIQUE_CONSTRAINT_NAME`, kcu.`REFERENCED_TABLE_NAME`, kcu.`COLUMN_NAME`, kcu.`REFERENCED_COLUMN_NAME`FROM information_schema.`TABLE_CONSTRAINTS` tc LEFT JOIN information_schema.`CHECK_CONSTRAINTS` cc ON tc.`CONSTRAINT_SCHEMA` = cc.`CONSTRAINT_SCHEMA` AND tc.`TABLE_NAME` = cc.`TABLE_NAME` AND tc.`CONSTRAINT_TYPE` = 'CHECK' LEFT JOIN information_schema.`REFERENTIAL_CONSTRAINTS` rc ON tc.`CONSTRAINT_SCHEMA` = rc.`CONSTRAINT_SCHEMA` AND tc.`TABLE_NAME` = rc.`TABLE_NAME` AND tc.`CONSTRAINT_TYPE` = 'UNIQUE' LEFT JOIN information_schema.`KEY_COLUMN_USAGE` kcu ON tc.`CONSTRAINT_SCHEMA` = kcu.`CONSTRAINT_SCHEMA` AND tc.`TABLE_NAME` = kcu.`TABLE_NAME` AND (tc.`CONSTRAINT_TYPE` = 'FOREIGN KEY' OR tc.`CONSTRAINT_TYPE` = 'UNIQUE') AND kcu.`CONSTRAINT_NAME` = tc.`CONSTRAINT_NAME` AND LOWER(kcu.`COLUMN_NAME`) != 'row_end' WHERE tc.`TABLE_SCHEMA` = '")
.append(databaseName)
.append("' AND tc.`TABLE_NAME` = '")
.append(tableName)
.append("'");
- log.trace("statement={}", statement);
+ log.trace("mapped obtain table constraint metadata statement {} to prepared statement", statement);
try {
return connection.prepareStatement(statement.toString());
} catch (SQLException e) {
- log.error("Failed to prepare statement {}, reason: {}", statement, e.getMessage());
- throw new QueryMalformedException("Failed to prepare statement", e);
+ log.error("Failed to prepare statement {}: {}", statement, e.getMessage());
+ throw new QueryMalformedException("Failed to prepare statement: " + e.getMessage(), e);
}
}
@@ -841,7 +839,7 @@ public interface QueryMapper {
return connection.prepareStatement(statement.toString());
} catch (SQLException e) {
log.error("Failed to prepare statement {}: {}", statement, e.getMessage());
- throw new QueryMalformedException("Failed to prepare statement", e);
+ throw new QueryMalformedException("Failed to prepare statement: " + e.getMessage(), e);
}
}
@@ -864,6 +862,11 @@ public interface QueryMapper {
if (data == null) {
return null;
}
+ /* boolean encoding fix */
+ if (column.getColumnType().equals(TableColumnType.TINYINT) && column.getSize() == 1) {
+ log.debug("column {} is of type tinyint with size {}: map to boolean", column.getInternalName(), column.getSize());
+ column.setColumnType(TableColumnType.BOOL);
+ }
switch (column.getColumnType()) {
case DATE -> {
if (column.getDateFormat() == null) {
@@ -902,7 +905,7 @@ public interface QueryMapper {
log.trace("mapping {} -> biginteger", data);
return new BigInteger(String.valueOf(data));
}
- case INT, TINYINT, SMALLINT, MEDIUMINT -> {
+ case INT, SMALLINT, MEDIUMINT, TINYINT -> {
log.trace("mapping {} -> integer", data);
return Integer.parseInt(String.valueOf(data));
}
@@ -949,37 +952,6 @@ public interface QueryMapper {
}
}
- default String generateInsertFromTemporaryTableSQL(Table table) {
- final StringBuilder statement = new StringBuilder("INSERT INTO `")
- .append(table.getDatabase().getInternalName())
- .append("`.`")
- .append(table.getInternalName())
- .append("` SELECT ");
- for (TableColumn tc : table.getColumns()) {
- statement.append("`");
- statement.append(tc.getInternalName()).append("`,");
- }
- statement.deleteCharAt(statement.length() - 1);
- statement.append(" FROM `")
- .append(table.getDatabase().getInternalName())
- .append("`.`")
- .append(table.getInternalName())
- .append("_temporary`");
-
- statement.append(" ON DUPLICATE KEY UPDATE ");
- for (TableColumn tc : table.getColumns())
- statement.append("`")
- .append(tc.getInternalName())
- .append("`")
- .append("=")
- .append("VALUES(`")
- .append(tc.getInternalName())
- .append("`),");
- statement.deleteCharAt(statement.length() - 1);
- statement.append(";");
- return statement.toString();
- }
-
default void prepareStatementWithColumnTypeObject(PreparedStatement ps, TableColumnType columnType, int idx, Object value) throws SQLException {
switch (columnType) {
case BLOB, TINYBLOB, MEDIUMBLOB, LONGBLOB:
diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/TableMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/TableMapper.java
index e74b529fbfde23f3641a568ea1a025757f6f9c20..b3b734d75e3202c5c6fc31704ec12fd97a0bc5e6 100644
--- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/TableMapper.java
+++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/TableMapper.java
@@ -85,7 +85,8 @@ public interface TableMapper {
ColumnTypeDto columnTypeToColumnTypeDto(TableColumnType data);
@Mappings({
- @Mapping(target = "constraints", ignore = true)
+ @Mapping(target = "constraints", ignore = true),
+ @Mapping(target = "processedConstraints", expression = "java(false)"),
})
Table tableCreateDtoToTable(TableCreateDto data);
@@ -124,6 +125,7 @@ public interface TableMapper {
default Unique columnNameListToUnique(Table table, List<String> names) throws TableMalformedException {
return Unique.builder()
.table(table)
+ .name("UK_" + String.join("_", names))
.columns(columnNameListToTableColumn(table, names))
.build();
}
@@ -131,7 +133,7 @@ public interface TableMapper {
ReferenceType referenceTypeDtoToReferenceType(ReferenceTypeDto dto);
@Transactional(readOnly = true)
- default ForeignKey foreignKeyCreateDtoToForeignKey(Table table, ForeignKeyCreateDto data) throws TableMalformedException {
+ default ForeignKey foreignKeyCreateDtoToForeignKey(Table table, ForeignKeyCreateDto data, Integer index) throws TableMalformedException {
final String referencedTableInternalName = nameToInternalName(data.getReferencedTable());
final Optional<Table> optional = table.getDatabase()
.getTables()
@@ -142,11 +144,13 @@ public interface TableMapper {
log.error("Failed to find referenced table with internal name {} in database with id {}", referencedTableInternalName, table.getDatabase().getId());
throw new TableMalformedException("Failed to find referenced table with internal name " + referencedTableInternalName + " in database with id " + table.getDatabase().getId());
}
- final ForeignKey.ForeignKeyBuilder builder = ForeignKey.builder()
+ final ForeignKey foreignKey = ForeignKey.builder()
+ .name("fk_" + table.getInternalName() + "_" + (index + 1))
.table(table)
.onUpdate(referenceTypeDtoToReferenceType(data.getOnUpdate()))
.onDelete(referenceTypeDtoToReferenceType(data.getOnDelete()))
- .referencedTable(optional.get());
+ .referencedTable(optional.get())
+ .build();
final List<TableColumn> columns = columnNameListToTableColumn(table, data.getColumns());
final List<TableColumn> referencedColumns = columnNameListToTableColumn(optional.get(), data.getReferencedColumns());
if (columns.isEmpty()) {
@@ -158,7 +162,7 @@ public interface TableMapper {
throw new TableMalformedException("There have to be equally as many columns and referenced columns in a foreign key");
}
final List<ForeignKeyReference> references = new ArrayList<>();
- final ForeignKey foreignKey = builder.references(references).build();
+ foreignKey.setReferences(references);
for (int i = 0; i < columns.size(); i++) {
TableColumn column = columns.get(i);
TableColumn referencedColumn = referencedColumns.get(i);
@@ -177,44 +181,43 @@ public interface TableMapper {
if (data == null) {
return null;
}
-
- ForeignKeyDto dto = new ForeignKeyDto(
- new ArrayList<>(),
- tableToTableBriefDto(data.getReferencedTable()),
- new ArrayList<>(),
- referenceTypeDtoToReferenceType(data.getOnUpdate()),
- referenceTypeDtoToReferenceType(data.getOnDelete())
- );
-
+ final ForeignKeyDto foreignKey = ForeignKeyDto.builder()
+ .name(data.getName())
+ .columns(new LinkedList<>())
+ .referencedColumns(new LinkedList<>())
+ .referencedTable(tableToTableBriefDto(data.getReferencedTable()))
+ .onDelete(referenceTypeDtoToReferenceType(data.getOnDelete()))
+ .onUpdate(referenceTypeDtoToReferenceType(data.getOnUpdate()))
+ .build();
for (ForeignKeyReference reference : data.getReferences()) {
- dto.getColumns().add(tableColumnToColumnDto(reference.getColumn()));
- dto.getReferencedColumns().add(tableColumnToColumnDto(reference.getReferencedColumn()));
+ foreignKey.getColumns().add(tableColumnToColumnDto(reference.getColumn()));
+ foreignKey.getReferencedColumns().add(tableColumnToColumnDto(reference.getReferencedColumn()));
}
- return dto;
+ return foreignKey;
}
+ @Transactional(readOnly = true)
default Constraints constraintsCreateDtoToConstraints(Table table, ConstraintsCreateDto data)
throws TableMalformedException {
if (data == null) {
return null;
}
- Constraints.ConstraintsBuilder builder = Constraints.builder();
+ final Constraints.ConstraintsBuilder builder = Constraints.builder();
if (data.getUniques() != null) {
- List<Unique> uniques = new ArrayList<>();
+ final List<Unique> uniques = new ArrayList<>();
for (List<String> columns : data.getUniques()) {
uniques.add(columnNameListToUnique(table, columns));
}
builder.uniques(uniques);
}
if (data.getForeignKeys() != null) {
- List<ForeignKey> foreignKeys = new ArrayList<>();
- for (ForeignKeyCreateDto foreignKeyData : data.getForeignKeys()) {
- foreignKeys.add(foreignKeyCreateDtoToForeignKey(table, foreignKeyData));
+ final List<ForeignKey> foreignKeys = new ArrayList<>();
+ for (int i = 0; i < data.getForeignKeys().size(); i++) {
+ foreignKeys.add(foreignKeyCreateDtoToForeignKey(table, data.getForeignKeys().get(i), i));
}
builder.foreignKeys(foreignKeys);
}
-
return builder.build();
}
@@ -588,7 +591,10 @@ public interface TableMapper {
.name(resultSet.getString(10))
.internalName(resultSet.getString(10))
.build();
- if (resultSet.getString(5) != null) {
+ /* fix boolean and set size for others */
+ if (resultSet.getString(8).equalsIgnoreCase("tinyint(1)")) {
+ column.setColumnType(TableColumnType.BOOL);
+ } else if (resultSet.getString(5) != null) {
column.setSize(resultSet.getLong(5));
} else if (resultSet.getString(6) != null) {
column.setSize(resultSet.getLong(6));
diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/ViewMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/ViewMapper.java
index d1fdb10b8f34de7a523505f98f1d95af7439f647..c63098a06f964a2cc28a15f8f28910809ad5d0c9 100644
--- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/ViewMapper.java
+++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/ViewMapper.java
@@ -11,6 +11,7 @@ import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
import org.mapstruct.Mappings;
import org.mapstruct.Named;
+import org.springframework.transaction.annotation.Transactional;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -48,7 +49,13 @@ public interface ViewMapper {
ViewBriefDto viewToViewBriefDto(View data);
- TableColumn viewColumnToTableColumn(ViewColumn data);
+ @Transactional(readOnly = true)
+ default TableColumn viewColumnToTableColumn(ViewColumn data) {
+ return data.getColumn()
+ .toBuilder()
+ .alias(data.getAlias())
+ .build();
+ }
default List<ViewColumn> tableColumnsToViewColumns(View view, List<TableColumn> data) {
final int[] idx = new int[]{0};
@@ -69,7 +76,7 @@ public interface ViewMapper {
view.getColumns()
.forEach(c -> statement.append(idx[0]++ > 0 ? "," : "")
.append("`")
- .append(c.getColumn().getInternalName())
+ .append(c.getAlias() != null ? c.getAlias() : c.getColumn().getInternalName())
.append("`"));
statement.append(" FROM `")
.append(view.getInternalName())
diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableColumnEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableColumnEndpoint.java
index 47193508ba32c0cd0ee33091b5860f7c3380b5c7..acd980cc2dfb256129414a11278ef0feea5c1489 100644
--- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableColumnEndpoint.java
+++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableColumnEndpoint.java
@@ -6,7 +6,7 @@ import at.tuwien.api.error.ApiErrorDto;
import at.tuwien.entities.database.table.columns.TableColumn;
import at.tuwien.exception.*;
import at.tuwien.mapper.TableMapper;
-import at.tuwien.service.TableService;
+import at.tuwien.service.TableColumnService;
import at.tuwien.utils.PrincipalUtil;
import at.tuwien.utils.UserUtil;
import at.tuwien.validation.EndpointValidator;
@@ -35,14 +35,15 @@ import java.security.Principal;
public class TableColumnEndpoint {
private final TableMapper tableMapper;
- private final TableService tableService;
private final EndpointValidator endpointValidator;
+ private final TableColumnService tableColumnService;
@Autowired
- public TableColumnEndpoint(TableMapper tableMapper, TableService tableService, EndpointValidator endpointValidator) {
+ public TableColumnEndpoint(TableMapper tableMapper, EndpointValidator endpointValidator,
+ TableColumnService tableColumnService) {
this.tableMapper = tableMapper;
- this.tableService = tableService;
this.endpointValidator = endpointValidator;
+ this.tableColumnService = tableColumnService;
}
@PutMapping
@@ -76,8 +77,7 @@ public class TableColumnEndpoint {
@NotNull @PathVariable("tableId") Long tableId,
@NotNull @PathVariable("columnId") Long columnId,
@NotNull @Valid @RequestBody ColumnSemanticsUpdateDto updateDto,
- @NotNull Principal principal,
- @NotNull @RequestHeader("Authorization") String authorization)
+ @NotNull Principal principal)
throws TableNotFoundException, TableMalformedException, DatabaseNotFoundException, NotAllowedException,
AccessDeniedException {
log.debug("endpoint update table, id={}, tableId={}, columnId={}, {}", id, tableId, columnId, PrincipalUtil.formatForDebug(principal));
@@ -85,11 +85,12 @@ public class TableColumnEndpoint {
endpointValidator.validateOnlyAccess(id, principal, true);
endpointValidator.validateOnlyOwnerOrWriteAll(id, tableId, principal);
}
- final TableColumn column = tableService.update(id, tableId, columnId, updateDto, authorization);
+ final TableColumn column = tableColumnService.update(id, tableId, columnId, updateDto);
log.info("Updated table semantics of table with id {} and database with id {}", tableId, id);
- final ColumnDto dto = tableMapper.tableColumnToColumnDto(column);
+ final ColumnDto columnDto = tableMapper.tableColumnToColumnDto(column);
+ log.trace("find table data resulted in column {}", columnDto);
return ResponseEntity.accepted()
- .body(dto);
+ .body(columnDto);
}
}
diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java
index 73cf58157eea48134f6c716400de5e67ee84249d..a06d7987f0a46840ada94bc2d9ab9138b37a9bc8 100644
--- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java
+++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableDataEndpoint.java
@@ -173,6 +173,14 @@ public class TableDataEndpoint {
log.debug("endpoint insert data from csv, databaseId={}, tableId={}, data={}, {}", databaseId, tableId, data, PrincipalUtil.formatForDebug(principal));
/* check */
endpointValidator.validateOnlyWriteOwnOrWriteAllAccess(databaseId, tableId, principal);
+ if (data.getNullElement() == null) {
+ log.debug("null element not present, default to empty string");
+ data.setNullElement("");
+ }
+ if (data.getLineTermination() == null) {
+ log.debug("line termination not present, default to \\r\\n");
+ data.setLineTermination("\r\n");
+ }
/* insert */
queryService.insert(databaseId, tableId, data, principal);
return ResponseEntity.accepted()
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java
index fab3add8f6bc3b103fd9b59d5cbd162ae5903b80..038089a720f04c23715f8931b16a605c8bf933a1 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/config/MariaDbConfig.java
@@ -327,7 +327,7 @@ public class MariaDbConfig {
}
public static ColumnTypeDto typetoColumnTypeDto(String data) throws Exception {
- if (data.toUpperCase().startsWith("TINYINT(1)")) {
+ if (data.equalsIgnoreCase("TINYINT(1)")) {
/* boolean in MySQL */
return ColumnTypeDto.BOOL;
}
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableColumnEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableColumnEndpointUnitTest.java
index 5ed2d91817748146788778f2a9a538a83bb9d312..697fd5dff2816e725e111018977c461806272db1 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableColumnEndpointUnitTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableColumnEndpointUnitTest.java
@@ -12,6 +12,7 @@ import at.tuwien.entities.database.table.columns.TableColumn;
import at.tuwien.exception.*;
import at.tuwien.service.AccessService;
import at.tuwien.service.DatabaseService;
+import at.tuwien.service.TableColumnService;
import at.tuwien.service.TableService;
import lombok.extern.log4j.Log4j2;
import org.junit.jupiter.api.Test;
@@ -50,6 +51,9 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest {
@MockBean
private TableService tableService;
+ @MockBean
+ private TableColumnService tableColumnService;
+
@Autowired
private TableColumnEndpoint tableColumnEndpoint;
@@ -95,8 +99,7 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest {
@Test
@WithMockUser(username = USER_1_USERNAME, authorities = {"modify-table-column-semantics"})
public void update_publicHasRoleHasOwnWriteAccess_succeeds() throws TableNotFoundException, NotAllowedException,
- TableMalformedException, DatabaseNotFoundException, ContainerNotFoundException,
- SemanticEntityPersistException, SemanticEntityNotFoundException, QueryMalformedException, at.tuwien.exception.AccessDeniedException {
+ TableMalformedException, DatabaseNotFoundException, at.tuwien.exception.AccessDeniedException {
final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder()
.unitUri(UNIT_MILLIMETRE_URI)
.build();
@@ -148,8 +151,7 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest {
@WithMockUser(username = USER_2_USERNAME, authorities = {"modify-table-column-semantics"})
public void update_publicHasRoleForeignHasAllWriteAccess_succeeds() throws TableNotFoundException,
NotAllowedException, TableMalformedException, DatabaseNotFoundException,
- ContainerNotFoundException, SemanticEntityPersistException, SemanticEntityNotFoundException,
- QueryMalformedException, at.tuwien.exception.AccessDeniedException {
+ at.tuwien.exception.AccessDeniedException {
final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder()
.unitUri(UNIT_MILLIMETRE_URI)
.build();
@@ -204,9 +206,7 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest {
@Test
@WithMockUser(username = USER_1_USERNAME, authorities = {"modify-table-column-semantics"})
public void update_privateHasRoleHasOwnWriteAccess_succeeds() throws TableNotFoundException, NotAllowedException,
- TableMalformedException, DatabaseNotFoundException, ContainerNotFoundException,
- SemanticEntityPersistException, SemanticEntityNotFoundException, QueryMalformedException,
- at.tuwien.exception.AccessDeniedException {
+ TableMalformedException, DatabaseNotFoundException, at.tuwien.exception.AccessDeniedException {
final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder()
.unitUri(UNIT_MILLIMETRE_URI)
.build();
@@ -258,8 +258,7 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest {
@WithMockUser(username = USER_2_USERNAME, authorities = {"modify-table-column-semantics"})
public void update_privateHasRoleForeignHasAllWriteAccess_succeeds() throws TableNotFoundException,
NotAllowedException, TableMalformedException, DatabaseNotFoundException,
- ContainerNotFoundException, SemanticEntityPersistException, SemanticEntityNotFoundException,
- QueryMalformedException, at.tuwien.exception.AccessDeniedException {
+ at.tuwien.exception.AccessDeniedException {
final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder()
.unitUri(UNIT_MILLIMETRE_URI)
.build();
@@ -277,8 +276,7 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest {
ColumnSemanticsUpdateDto data, UUID userId,
Principal principal, DatabaseAccess access)
throws DatabaseNotFoundException, NotAllowedException, TableNotFoundException, TableMalformedException,
- ContainerNotFoundException, SemanticEntityPersistException, SemanticEntityNotFoundException,
- QueryMalformedException, at.tuwien.exception.AccessDeniedException {
+ at.tuwien.exception.AccessDeniedException {
/* mock */
if (database != null) {
@@ -292,12 +290,12 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest {
if (table != null) {
when(tableService.find(databaseId, tableId))
.thenReturn(table);
- when(tableService.update(databaseId, tableId, columnId, data, "abc"))
+ when(tableColumnService.update(databaseId, tableId, columnId, data))
.thenReturn(column);
} else {
doThrow(TableNotFoundException.class)
- .when(tableService)
- .update(databaseId, tableId, columnId, data, "abc");
+ .when(tableColumnService)
+ .update(databaseId, tableId, columnId, data);
doThrow(TableNotFoundException.class)
.when(tableService)
.find(databaseId, tableId);
@@ -312,6 +310,6 @@ public class TableColumnEndpointUnitTest extends BaseUnitTest {
}
/* test */
- return tableColumnEndpoint.update(databaseId, tableId, columnId, data, principal, "abc");
+ return tableColumnEndpoint.update(databaseId, tableId, columnId, data, principal);
}
}
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java
index 933d4ba78aefde918ecbbaa7f4f63fb5c5249871..a866e8240f6c71e5e3e1258d696eaf0986f667f6 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/endpoints/TableDataEndpointUnitTest.java
@@ -3,6 +3,7 @@ package at.tuwien.endpoints;
import at.tuwien.BaseUnitTest;
import at.tuwien.SortType;
import at.tuwien.annotations.MockAmqp;
+import at.tuwien.annotations.MockListeners;
import at.tuwien.annotations.MockOpensearch;
import at.tuwien.api.database.query.ImportDto;
import at.tuwien.api.database.query.QueryResultDto;
@@ -36,6 +37,7 @@ import java.util.UUID;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.when;
@Log4j2
@@ -43,6 +45,7 @@ import static org.mockito.Mockito.when;
@ExtendWith(SpringExtension.class)
@MockAmqp
@MockOpensearch
+@MockListeners
public class TableDataEndpointUnitTest extends BaseUnitTest {
@MockBean
@@ -469,7 +472,7 @@ public class TableDataEndpointUnitTest extends BaseUnitTest {
.thenReturn(table);
when(accessService.find(databaseId, userId))
.thenReturn(access);
- when(queryService.tableFindAll(databaseId, tableId, timestamp, page, size, principal))
+ when(queryService.tableFindAll(eq(databaseId), eq(tableId), eq(timestamp), anyLong(), anyLong(), eq(principal)))
.thenReturn(QUERY_1_RESULT_DTO);
/* test */
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
index 94a97d3643140ef3de727720821c1e1b37f94bb0..26a9b506e5aa3809c26646fe5f4abbc27ac36527 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java
@@ -590,7 +590,7 @@ public class PrometheusEndpointMvcTest extends BaseUnitTest {
/* mock */
try {
- tableColumnEndpoint.update(DATABASE_1_ID, TABLE_1_ID, TABLE_1_COLUMNS.get(3).getId(), request, USER_1_PRINCIPAL, "s3cr3t");
+ tableColumnEndpoint.update(DATABASE_1_ID, TABLE_1_ID, TABLE_1_COLUMNS.get(3).getId(), request, USER_1_PRINCIPAL);
} catch (Exception e) {
/* ignore */
}
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/repository/DatabaseRepositoryIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/repository/DatabaseRepositoryIntegrationTest.java
index 0e7473b849b560f28d93feb6705d26c366ab0125..537f5f9882c12607d43894518575e33c1e011572 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/repository/DatabaseRepositoryIntegrationTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/repository/DatabaseRepositoryIntegrationTest.java
@@ -50,6 +50,12 @@ public class DatabaseRepositoryIntegrationTest extends BaseUnitTest {
TABLE_5.setColumns(TABLE_5_COLUMNS);
TABLE_6.setColumns(TABLE_6_COLUMNS);
TABLE_7.setColumns(TABLE_7_COLUMNS);
+ DATABASE_1.setAccesses(List.of());
+ DATABASE_2.setAccesses(List.of());
+ VIEW_1.setColumns(VIEW_1_COLUMNS);
+ VIEW_2.setColumns(VIEW_2_COLUMNS);
+ VIEW_3.setColumns(VIEW_3_COLUMNS);
+ VIEW_4.setColumns(VIEW_4_COLUMNS);
/* metadata database */
imageRepository.save(IMAGE_1);
licenseRepository.save(LICENSE_1);
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java
index 89a87c5e260b9f4d4e671f9587d62f17045a54c6..38fd8f645f66a1e9a866788b7b653cb9eb1729cf 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/DatabaseServiceIntegrationTest.java
@@ -13,6 +13,10 @@ import at.tuwien.entities.database.View;
import at.tuwien.entities.database.table.Table;
import at.tuwien.entities.database.table.columns.TableColumn;
import at.tuwien.entities.database.table.columns.TableColumnType;
+import at.tuwien.entities.database.table.constraints.Constraints;
+import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKey;
+import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKeyReference;
+import at.tuwien.entities.database.table.constraints.unique.Unique;
import at.tuwien.entities.user.User;
import at.tuwien.exception.*;
import at.tuwien.repository.mdb.*;
@@ -341,11 +345,11 @@ public class DatabaseServiceIntegrationTest extends BaseUnitTest {
}
@Test
- public void obtainMetadata_tableWithoutVersioning_succeeds() throws QueryMalformedException,
+ public void obtainTablesMetadata_tableWithoutVersioning_succeeds() throws QueryMalformedException,
DatabaseNotFoundException, ColumnParseException {
/* test */
- final Database response = databaseService.obtainMetadata(DATABASE_1_ID);
+ final Database response = databaseService.obtainTablesMetadata(DATABASE_1_ID);
final List<Table> tables = response.getTables();
assertEquals(7, tables.size());
final Optional<Table> optional3 = tables.stream().filter(t -> t.getInternalName().equals("weather_aut_without_versioning")).findFirst();
@@ -360,11 +364,11 @@ public class DatabaseServiceIntegrationTest extends BaseUnitTest {
}
@Test
- public void obtainMetadata_tableWithVersioning_succeeds() throws QueryMalformedException, DatabaseNotFoundException,
+ public void obtainTablesMetadata_tableWithVersioning_succeeds() throws QueryMalformedException, DatabaseNotFoundException,
ColumnParseException {
/* test */
- final Database response = databaseService.obtainMetadata(DATABASE_1_ID);
+ final Database response = databaseService.obtainTablesMetadata(DATABASE_1_ID);
final List<Table> tables = response.getTables();
assertEquals(7, tables.size());
final Optional<Table> optional4 = tables.stream().filter(t -> t.getInternalName().equals("weather_aut")).findFirst();
@@ -380,11 +384,14 @@ public class DatabaseServiceIntegrationTest extends BaseUnitTest {
}
@Test
- public void obtainMetadata_view_succeeds() throws QueryMalformedException, DatabaseNotFoundException,
+ public void obtainViewsMetadata_view_succeeds() throws QueryMalformedException, DatabaseNotFoundException,
ColumnParseException {
+ /* mock */
+ databaseService.obtainTablesMetadata(DATABASE_1_ID); /* weather_aut is not yet in metadata-db */
+
/* test */
- final Database response = databaseService.obtainMetadata(DATABASE_1_ID);
+ final Database response = databaseService.obtainViewsMetadata(DATABASE_1_ID);
final List<Table> tables = response.getTables();
assertEquals(7, tables.size());
final List<View> views = response.getViews();
@@ -404,10 +411,61 @@ public class DatabaseServiceIntegrationTest extends BaseUnitTest {
assertColumn(view1.getColumns().get(1).getColumn(), 1, "date", TableColumnType.DATE, null, false, false, false);
}
+ @Test
+ public void obtainConstraints_inlineConstraints_succeeds() throws QueryMalformedException,
+ DatabaseNotFoundException, TableMalformedException, SQLException, ColumnParseException {
+
+ /* test */
+ generic_obtainConstraints("CREATE TABLE foreigner (id BIGINT PRIMARY KEY NOT NULL, weather_id BIGINT REFERENCES weather_aus (id), qty INT CHECK (qty > 0), firstname VARCHAR(255) UNIQUE) WITH SYSTEM VERSIONING;");
+ }
+
+ @Test
+ public void obtainConstraints_complexConstraints_succeeds() throws QueryMalformedException,
+ DatabaseNotFoundException, TableMalformedException, SQLException, ColumnParseException {
+
+ /* test */
+ generic_obtainConstraints("CREATE TABLE foreigner (id BIGINT NOT NULL, weather_id BIGINT NOT NULL, qty INT NOT NULL, firstname VARCHAR(255) NOT NULL, PRIMARY KEY (id), UNIQUE (firstname), FOREIGN KEY (weather_id) REFERENCES weather_aus (id), CONSTRAINT pos_qty CHECK (qty > 0)) WITH SYSTEM VERSIONING;");
+ }
+
/* ################################################################################################### */
/* ## GENERIC TEST CASES ## */
/* ################################################################################################### */
+ protected void generic_obtainConstraints(String sql) throws QueryMalformedException, DatabaseNotFoundException,
+ TableMalformedException, SQLException, ColumnParseException {
+
+ /* mock */
+ MariaDbConfig.execute(DATABASE_1, sql);
+ databaseService.obtainTablesMetadata(DATABASE_1_ID);
+
+ /* test */
+ final Database response = databaseService.obtainConstraints(DATABASE_1_ID);
+ final List<Table> tables = response.getTables();
+ assertEquals(8, tables.size());
+ final Optional<Table> optional8 = tables.stream().filter(t -> t.getInternalName().equals("foreigner")).findFirst();
+ assertTrue(optional8.isPresent());
+ final Table table8 = optional8.get();
+ assertNotNull(table8.getConstraints());
+ final Constraints constraints8 = table8.getConstraints();
+ assertNotNull(constraints8.getUniques());
+ assertEquals(1, constraints8.getUniques().size());
+ final Unique unique0 = constraints8.getUniques().get(0);
+ assertEquals("foreigner", unique0.getTable().getInternalName());
+ assertEquals(1, unique0.getColumns().size());
+ assertEquals("firstname", unique0.getColumns().get(0).getInternalName());
+ assertNotNull(constraints8.getChecks());
+ assertEquals(1, constraints8.getChecks().size());
+ assertNotNull(constraints8.getForeignKeys());
+ assertEquals(1, constraints8.getForeignKeys().size());
+ final ForeignKey foreignKey0 = constraints8.getForeignKeys().get(0);
+ assertEquals("foreigner", foreignKey0.getTable().getInternalName());
+ assertEquals("weather_aus", foreignKey0.getReferencedTable().getInternalName());
+ assertEquals(1, foreignKey0.getReferences().size());
+ final ForeignKeyReference foreignKeyReference0 = foreignKey0.getReferences().get(0);
+ assertEquals("weather_id", foreignKeyReference0.getColumn().getInternalName());
+ assertEquals("id", foreignKeyReference0.getReferencedColumn().getInternalName());
+ }
+
protected void generic_insert(String query, Long assertQueryId) throws SQLException, QueryMalformedException {
/* mock */
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceIntegrationTest.java
index 7ec7d0a78475e561b177dbc16d911a4d9d7a8bd5..185c7cda6191f756a84efdd959b70d4f4f32cd9f 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceIntegrationTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/MetadataServiceIntegrationTest.java
@@ -56,6 +56,9 @@ public class MetadataServiceIntegrationTest extends BaseUnitTest {
TABLE_2.setColumns(TABLE_2_COLUMNS);
TABLE_3.setColumns(TABLE_3_COLUMNS);
TABLE_4.setColumns(TABLE_4_COLUMNS);
+ VIEW_1.setColumns(VIEW_1_COLUMNS);
+ VIEW_2.setColumns(VIEW_2_COLUMNS);
+ VIEW_3.setColumns(VIEW_3_COLUMNS);
/* metadata database */
imageRepository.save(IMAGE_1);
userRepository.save(USER_1);
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java
index 77af8cb2fcd0e400b5ab5c5c0d60adad1f05993b..560fd4e89d6a8e5210a0bd0043fbe38dc7f6f52e 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/QueryServiceIntegrationTest.java
@@ -88,6 +88,9 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
@Autowired
private LicenseRepository licenseRepository;
+ @Autowired
+ private DatabaseService databaseService;
+
@MockBean
private DataDbSidecarGateway dataDbSidecarGateway;
@@ -105,7 +108,8 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
}
@BeforeEach
- public void beforeEach() throws SQLException {
+ public void beforeEach() throws SQLException, DatabaseUnchangedException, QueryMalformedException,
+ ColumnParseException, DatabaseNotFoundException, TableMalformedException {
TABLE_1.setColumns(TABLE_1_COLUMNS);
TABLE_2.setColumns(TABLE_2_COLUMNS);
TABLE_3.setColumns(TABLE_3_COLUMNS);
@@ -115,6 +119,10 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
TABLE_7.setColumns(TABLE_7_COLUMNS);
DATABASE_1.setAccesses(List.of());
DATABASE_2.setAccesses(List.of());
+ VIEW_1.setColumns(VIEW_1_COLUMNS);
+ VIEW_2.setColumns(VIEW_2_COLUMNS);
+ VIEW_3.setColumns(VIEW_3_COLUMNS);
+ VIEW_4.setColumns(VIEW_4_COLUMNS);
/* metadata database */
imageRepository.save(IMAGE_1);
licenseRepository.save(LICENSE_1);
@@ -125,6 +133,12 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
MariaDbConfig.dropAllDatabases(CONTAINER_1);
MariaDbConfig.createInitDatabase(CONTAINER_1, DATABASE_1);
MariaDbConfig.createInitDatabase(CONTAINER_1, DATABASE_2);
+ databaseService.obtainTablesMetadata(DATABASE_1_ID);
+ databaseService.obtainTablesMetadata(DATABASE_2_ID);
+ databaseService.obtainConstraints(DATABASE_1_ID);
+ databaseService.obtainConstraints(DATABASE_2_ID);
+ databaseService.obtainViewsMetadata(DATABASE_1_ID);
+ databaseService.obtainViewsMetadata(DATABASE_2_ID);
}
@Test
@@ -134,7 +148,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
/* test */
final QueryResultDto result = queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, Instant.now(),
- null, null, USER_1_PRINCIPAL);
+ 0L, 10L, USER_1_PRINCIPAL);
assertEquals(3, result.getResult().size());
assertEquals(BigInteger.valueOf(1L), result.getResult().get(0).get(TABLE_1_COLUMNS.get(0).getInternalName()));
assertEquals(toInstant("2008-12-01"), result.getResult().get(0).get(TABLE_1_COLUMNS.get(1).getInternalName()));
@@ -195,6 +209,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
.nullElement("NA")
.separator(';')
.location(filename)
+ .lineTermination("\r\n")
.build();
/* mock */
@@ -319,7 +334,7 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
DatabaseNotFoundException, ImageNotSupportedException, QueryMalformedException {
/* test */
- queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, null, null, null, USER_1_PRINCIPAL);
+ queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, null, 0L, 10L, USER_1_PRINCIPAL);
}
@Test
@@ -328,8 +343,8 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
final Instant timestamp = DATABASE_1_CREATED.minus(1, ChronoUnit.SECONDS);
/* test */
- queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, timestamp, null, null, USER_1_PRINCIPAL);
- queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, timestamp, null, null, USER_1_PRINCIPAL);
+ queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, timestamp, 0L, 10L, USER_1_PRINCIPAL);
+ queryService.tableFindAll(DATABASE_1_ID, TABLE_1_ID, timestamp, 0L, 10L, USER_1_PRINCIPAL);
}
@Test
@@ -452,22 +467,31 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
assertEquals(9L, response.getResultNumber());
assertNotNull(response.getResult());
final List<Map<String, Object>> result = response.getResult();
+ assertEquals(2, result.get(0).keySet().size());
assertEquals("Albury", result.get(0).get("a"));
assertEquals("Albury", result.get(0).get("location"));
+ assertEquals(2, result.get(1).keySet().size());
assertEquals("Albury", result.get(1).get("a"));
assertEquals("Albury", result.get(1).get("location"));
+ assertEquals(2, result.get(2).keySet().size());
assertEquals("Albury", result.get(2).get("a"));
assertEquals("Albury", result.get(2).get("location"));
+ assertEquals(2, result.get(3).keySet().size());
assertEquals("Albury", result.get(3).get("a"));
assertEquals("Sydney", result.get(3).get("location"));
+ assertEquals(2, result.get(4).keySet().size());
assertEquals("Albury", result.get(4).get("a"));
assertEquals("Sydney", result.get(4).get("location"));
+ assertEquals(2, result.get(5).keySet().size());
assertEquals("Albury", result.get(5).get("a"));
assertEquals("Sydney", result.get(5).get("location"));
+ assertEquals(2, result.get(6).keySet().size());
assertEquals("Albury", result.get(6).get("a"));
assertEquals("Vienna", result.get(6).get("location"));
+ assertEquals(2, result.get(7).keySet().size());
assertEquals("Albury", result.get(7).get("a"));
assertEquals("Vienna", result.get(7).get("location"));
+ assertEquals(2, result.get(8).keySet().size());
assertEquals("Albury", result.get(8).get("a"));
assertEquals("Vienna", result.get(8).get("location"));
}
@@ -523,18 +547,18 @@ public class QueryServiceIntegrationTest extends BaseUnitTest {
/* ordering */
final String[] keys = result.get(0).keySet().toArray(new String[0]);
assertEquals("date", keys[0]);
- assertEquals("rainfall", keys[1]);
- assertEquals("location", keys[2]);
+ assertEquals("loc", keys[1]);
+ assertEquals("rainfall", keys[2]);
assertEquals("mintemp", keys[3]);
/* values */
assertEquals(0.6, result.get(0).get("rainfall"));
- assertEquals("Albury", result.get(0).get("location"));
+ assertEquals("Albury", result.get(0).get("loc"));
assertEquals(13.4, result.get(0).get("mintemp"));
assertEquals(0.0, result.get(1).get("rainfall"));
- assertEquals("Albury", result.get(1).get("location"));
+ assertEquals("Albury", result.get(1).get("loc"));
assertEquals(7.4, result.get(1).get("mintemp"));
assertEquals(0.0, result.get(2).get("rainfall"));
- assertEquals("Albury", result.get(2).get("location"));
+ assertEquals("Albury", result.get(2).get("loc"));
assertEquals(12.9, result.get(2).get("mintemp"));
}
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableColumnServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableColumnServiceIntegrationTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..2391a5229007710d4d175448707ecd022190004e
--- /dev/null
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableColumnServiceIntegrationTest.java
@@ -0,0 +1,107 @@
+package at.tuwien.service;
+
+import at.tuwien.BaseUnitTest;
+import at.tuwien.annotations.MockAmqp;
+import at.tuwien.annotations.MockListeners;
+import at.tuwien.annotations.MockOpensearch;
+import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto;
+import at.tuwien.config.MariaDbConfig;
+import at.tuwien.config.MariaDbContainerConfig;
+import at.tuwien.entities.database.table.columns.TableColumn;
+import at.tuwien.entities.database.table.columns.TableColumnConcept;
+import at.tuwien.exception.*;
+import at.tuwien.repository.mdb.*;
+import lombok.extern.log4j.Log4j2;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.transaction.annotation.Transactional;
+import org.testcontainers.containers.MariaDBContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.sql.SQLException;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+@Log4j2
+@Testcontainers
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
+@EnableAutoConfiguration(exclude = RabbitAutoConfiguration.class)
+@SpringBootTest
+@ExtendWith(SpringExtension.class)
+@MockAmqp
+@MockListeners
+@MockOpensearch
+public class TableColumnServiceIntegrationTest extends BaseUnitTest {
+
+ @Autowired
+ private ImageRepository imageRepository;
+
+ @Autowired
+ private ContainerRepository containerRepository;
+
+ @Autowired
+ private DatabaseRepository databaseRepository;
+
+ @Autowired
+ private LicenseRepository licenseRepository;
+
+ @Autowired
+ private UserRepository userRepository;
+
+ @Autowired
+ private TableColumnService tableColumnService;
+
+ @Container
+ private static MariaDBContainer<?> mariaDBContainer = MariaDbContainerConfig.getContainer();
+
+ @BeforeEach
+ public void beforeEach() throws SQLException {
+ TABLE_1.setColumns(TABLE_1_COLUMNS);
+ TABLE_1_FOREIGN_KEY_1.setReferences(List.of(TABLE_1_FOREIGN_KEY_REFERENCE));
+ TABLE_1.setConstraints(TABLE_1_CONSTRAINTS);
+ TABLE_2.setColumns(TABLE_2_COLUMNS);
+ TABLE_2.setConstraints(TABLE_2_CONSTRAINTS);
+ TABLE_3.setColumns(TABLE_3_COLUMNS);
+ TABLE_3.setConstraints(TABLE_3_CONSTRAINTS);
+ TABLE_4.setColumns(TABLE_4_COLUMNS);
+ DATABASE_1.setAccesses(List.of());
+ TABLE_1.setDatabase(DATABASE_1);
+ TABLE_2.setDatabase(DATABASE_1);
+ TABLE_3.setDatabase(DATABASE_1);
+ TABLE_4.setDatabase(DATABASE_1);
+ /* metadata database */
+ imageRepository.save(IMAGE_1);
+ licenseRepository.save(LICENSE_1);
+ userRepository.saveAll(List.of(USER_1, USER_2));
+ containerRepository.save(CONTAINER_1);
+ databaseRepository.save(DATABASE_1);
+ /* data stuff */
+ MariaDbConfig.dropAllDatabases(CONTAINER_1);
+ MariaDbConfig.createInitDatabase(CONTAINER_1, DATABASE_1);
+ }
+
+ @Test
+ @Transactional
+ public void update_succeeds() throws TableNotFoundException, TableMalformedException, DatabaseNotFoundException {
+ final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder()
+ .conceptUri(COLUMN_CONCEPT_PRECIPITATION_URI)
+ .build();
+
+ /* test */
+ final TableColumn response = tableColumnService.update(DATABASE_1_ID, TABLE_1_ID, TABLE_1_COLUMNS.get(0).getId(),
+ request);
+ assertNotNull(response.getConcept());
+ final TableColumnConcept concept = response.getConcept();
+ assertEquals(COLUMN_CONCEPT_PRECIPITATION_URI, concept.getUri());
+ }
+
+}
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationReadTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationReadTest.java
index 4220af3f9ccb5515e5f1c79c0ad29de411dc00fe..c5a2400daf731d5d7e673b18c8f425fad67b44ce 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationReadTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationReadTest.java
@@ -98,7 +98,7 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest {
}
@Test
- public void findById_succeeds() throws TableNotFoundException, DatabaseNotFoundException{
+ public void findById_succeeds() throws TableNotFoundException, DatabaseNotFoundException {
/* test */
final Table response = tableService.find(DATABASE_1_ID, TABLE_1_ID);
@@ -126,8 +126,8 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest {
}
@Test
- public void findHistory_anonymous_succeeds() throws UserNotFoundException, TableNotFoundException,
- QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException {
+ public void findHistory_anonymous_succeeds() throws TableNotFoundException, QueryStoreException,
+ QueryMalformedException, DatabaseNotFoundException {
/* test */
final List<TableHistoryDto> response = tableService.findHistory(DATABASE_1_ID, TABLE_1_ID, null);
@@ -138,8 +138,8 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest {
@Test
@WithAnonymousUser
- public void findHistory_anonymous2_succeeds() throws UserNotFoundException, TableNotFoundException,
- QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException {
+ public void findHistory_anonymous2_succeeds()throws TableNotFoundException, QueryStoreException,
+ QueryMalformedException, DatabaseNotFoundException {
/* test */
final List<TableHistoryDto> response = tableService.findHistory(DATABASE_1_ID, TABLE_1_ID, null);
@@ -150,8 +150,8 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest {
@Test
@WithMockUser(username = USER_1_USERNAME, roles = {"RESEARCHER"})
- public void findHistory_researcher_succeeds() throws UserNotFoundException, TableNotFoundException,
- QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException {
+ public void findHistory_researcher_succeeds() throws TableNotFoundException, QueryStoreException,
+ QueryMalformedException, DatabaseNotFoundException {
/* test */
final List<TableHistoryDto> response = tableService.findHistory(DATABASE_1_ID, TABLE_1_ID, USER_1_PRINCIPAL);
@@ -162,8 +162,8 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest {
@Test
@WithMockUser(username = USER_2_USERNAME, roles = {"DEVELOPER"})
- public void findHistory_developer_succeeds() throws UserNotFoundException, TableNotFoundException,
- QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException {
+ public void findHistory_developer_succeeds()throws TableNotFoundException, QueryStoreException,
+ QueryMalformedException, DatabaseNotFoundException {
/* test */
final List<TableHistoryDto> response = tableService.findHistory(DATABASE_1_ID, TABLE_1_ID, USER_2_PRINCIPAL);
@@ -174,8 +174,8 @@ public class TableServiceIntegrationReadTest extends BaseUnitTest {
@Test
@WithMockUser(username = USER_3_USERNAME, roles = {"DATA_STEWARD"})
- public void findHistory_dataSteward_succeeds() throws UserNotFoundException, TableNotFoundException,
- QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException {
+ public void findHistory_dataSteward_succeeds() throws TableNotFoundException, QueryStoreException,
+ QueryMalformedException, DatabaseNotFoundException {
/* test */
final List<TableHistoryDto> response = tableService.findHistory(DATABASE_1_ID, TABLE_1_ID, USER_3_PRINCIPAL);
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationWriteTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationWriteTest.java
index c174c3e3b7fda1be798b8629b7363b6f394a82d6..685604fa7863db9ff3fef2269aaf339824c81ae2 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationWriteTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationWriteTest.java
@@ -6,16 +6,10 @@ import at.tuwien.annotations.MockListeners;
import at.tuwien.annotations.MockOpensearch;
import at.tuwien.api.database.table.TableCreateDto;
import at.tuwien.api.database.table.columns.ColumnCreateDto;
-import at.tuwien.api.database.table.columns.ColumnTypeDto;
-import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto;
-import at.tuwien.api.database.table.constraints.ConstraintsCreateDto;
import at.tuwien.config.MariaDbConfig;
import at.tuwien.config.MariaDbContainerConfig;
import at.tuwien.entities.database.table.Table;
import at.tuwien.entities.database.table.columns.TableColumn;
-import at.tuwien.entities.database.table.columns.TableColumnConcept;
-import at.tuwien.entities.identifier.Identifier;
-import at.tuwien.entities.identifier.IdentifierType;
import at.tuwien.exception.*;
import at.tuwien.mapper.TableMapper;
import at.tuwien.repository.mdb.*;
@@ -41,7 +35,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.junit.jupiter.api.Assertions.*;
-import static org.mockito.ArgumentMatchers.any;
@Log4j2
@Testcontainers
@@ -66,9 +59,6 @@ public class TableServiceIntegrationWriteTest extends BaseUnitTest {
@Autowired
private LicenseRepository licenseRepository;
- @Autowired
- private IdentifierRepository identifierRepository;
-
@Autowired
private TableService tableService;
@@ -124,9 +114,9 @@ public class TableServiceIntegrationWriteTest extends BaseUnitTest {
}
@Test
- public void create_withConstraints_succeeds() throws UserNotFoundException, TableMalformedException,
- QueryMalformedException, DatabaseNotFoundException, ImageNotSupportedException, TableNameExistsException,
- ContainerNotFoundException, SQLException, TableNotFoundException {
+ public void create_withConstraints_succeeds() throws TableMalformedException, QueryMalformedException,
+ DatabaseNotFoundException, ImageNotSupportedException, TableNameExistsException, SQLException,
+ TableNotFoundException {
/* test */
tableService.createTable(DATABASE_1_ID, TABLE_5_CREATE_DTO, USER_1_PRINCIPAL); // table to reference
@@ -178,7 +168,7 @@ public class TableServiceIntegrationWriteTest extends BaseUnitTest {
if (columnEntity.getInternalName().equals("id")) {
continue;
}
- log.trace("internalName={}, type={}", columnEntity.getInternalName(), columnEntity.getColumnType());
+ log.trace("internalName={}, type={}, size={}", columnEntity.getInternalName(), columnEntity.getColumnType(), columnEntity.getSize());
/* correct in the metadata database */
assertEquals(columnRequest.getNullAllowed(), columnEntity.getIsNullAllowed());
assertEquals(columnRequest.getPrimaryKey(), columnEntity.getIsPrimaryKey());
@@ -201,23 +191,6 @@ public class TableServiceIntegrationWriteTest extends BaseUnitTest {
});
}
- @Test
- @Transactional
- public void update_succeeds() throws TableNotFoundException, SemanticEntityPersistException,
- TableMalformedException, QueryMalformedException, DatabaseNotFoundException,
- SemanticEntityNotFoundException, ContainerNotFoundException {
- final ColumnSemanticsUpdateDto request = ColumnSemanticsUpdateDto.builder()
- .conceptUri(COLUMN_CONCEPT_PRECIPITATION_URI)
- .build();
-
- /* test */
- final TableColumn response = tableService.update(DATABASE_1_ID, TABLE_1_ID, TABLE_1_COLUMNS.get(0).getId(),
- request, "abc");
- assertNotNull(response.getConcept());
- final TableColumnConcept concept = response.getConcept();
- assertEquals(COLUMN_CONCEPT_PRECIPITATION_URI, concept.getUri());
- }
-
@Test
@Transactional
public void delete_succeeds() throws TableNotFoundException, TableMalformedException, QueryMalformedException,
@@ -230,8 +203,7 @@ public class TableServiceIntegrationWriteTest extends BaseUnitTest {
@Test
@Transactional
public void delete_full_succeeds() throws TableNotFoundException, TableMalformedException, QueryMalformedException,
- DatabaseNotFoundException, ImageNotSupportedException, UserNotFoundException, TableNameExistsException,
- ContainerNotFoundException {
+ DatabaseNotFoundException, ImageNotSupportedException, TableNameExistsException {
/* test */
final Table response = tableService.createTable(DATABASE_1_ID, TABLE_0_CREATE_DTO, USER_1_PRINCIPAL);
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java
index 50fa955f82d056b32610d20437b58e15bcb6363a..8ce238c0a02eb4b9052bb567db6babc01a64869d 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServiceIntegrationTest.java
@@ -15,6 +15,7 @@ import lombok.extern.log4j.Log4j2;
import org.junit.Rule;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.rules.Timeout;
@@ -92,7 +93,8 @@ public class ViewServiceIntegrationTest extends BaseUnitTest {
}
@BeforeEach
- public void beforeEach() {
+ public void beforeEach() throws DatabaseUnchangedException, QueryMalformedException, ColumnParseException,
+ DatabaseNotFoundException, TableMalformedException {
TABLE_1.setColumns(TABLE_1_COLUMNS);
TABLE_2.setColumns(TABLE_2_COLUMNS);
TABLE_3.setColumns(TABLE_3_COLUMNS);
@@ -102,6 +104,10 @@ public class ViewServiceIntegrationTest extends BaseUnitTest {
TABLE_7.setColumns(TABLE_7_COLUMNS);
DATABASE_1.setAccesses(List.of());
DATABASE_2.setAccesses(List.of());
+ VIEW_1.setColumns(VIEW_1_COLUMNS);
+ VIEW_2.setColumns(VIEW_2_COLUMNS);
+ VIEW_3.setColumns(VIEW_3_COLUMNS);
+ VIEW_4.setColumns(VIEW_4_COLUMNS);
/* metadata database */
imageRepository.save(IMAGE_1);
licenseRepository.save(LICENSE_1);
@@ -171,34 +177,4 @@ public class ViewServiceIntegrationTest extends BaseUnitTest {
assertNull(row2.get("lng"));
}
- @Test
- public void create_withAlias_succeeds() throws DatabaseNotFoundException, UserNotFoundException,
- DatabaseConnectionException, ViewMalformedException, QueryMalformedException {
- final ViewCreateDto request = ViewCreateDto.builder()
- .name(VIEW_2_NAME + "_with_alias")
- .query(VIEW_2_QUERY)
- .isPublic(VIEW_2_PUBLIC)
- .build();
-
- /* test */
- final View response = viewService.create(DATABASE_1_ID, request, USER_1_PRINCIPAL);
- assertEquals(VIEW_2_NAME + "_with_alias", response.getName());
- assertEquals(VIEW_2_INTERNAL_NAME + "_with_alias", response.getInternalName());
- assertEquals(VIEW_2_QUERY, response.getQuery());
- final List<ViewColumn> columns = response.getColumns();
- assertEquals(4, columns.size());
- final TableColumn column0 = columns.get(0).getColumn();
- assertEquals("date", column0.getInternalName());
- assertNull(column0.getAlias());
- final TableColumn column1 = columns.get(1).getColumn();
- assertEquals("location", column1.getInternalName());
- assertEquals("loc", column1.getAlias());
- final TableColumn column2 = columns.get(2).getColumn();
- assertEquals("rainfall", column2.getInternalName());
- assertNull(column2.getAlias());
- final TableColumn column3 = columns.get(3).getColumn();
- assertEquals("mintemp", column3.getInternalName());
- assertNull(column3.getAlias());
- }
-
}
diff --git a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServicePersistenceIntegrationTest.java b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServicePersistenceIntegrationTest.java
index 3480c1682a7cd304717f1c74db3864b788905b39..d57aad698548fd4687ab99b64f4adca2a4e7dc4b 100644
--- a/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServicePersistenceIntegrationTest.java
+++ b/dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/ViewServicePersistenceIntegrationTest.java
@@ -121,7 +121,7 @@ public class ViewServicePersistenceIntegrationTest extends BaseUnitTest {
assertEquals(VIEW_1_NAME, response.getName());
assertEquals(VIEW_1_INTERNAL_NAME, response.getInternalName());
assertEquals(VIEW_1_QUERY, response.getQuery());
- assertEquals(VIEW_1_COLUMNS.size(), response.getColumns().size());
+
}
}
diff --git a/dbrepo-metadata-service/rest-service/src/test/resources/csv/weather_aus_lastlinenull.csv b/dbrepo-metadata-service/rest-service/src/test/resources/csv/weather_aus_lastlinenull.csv
new file mode 100644
index 0000000000000000000000000000000000000000..12353bbaf7dc5468b5b6e4e904642a0c209ae21f
--- /dev/null
+++ b/dbrepo-metadata-service/rest-service/src/test/resources/csv/weather_aus_lastlinenull.csv
@@ -0,0 +1 @@
+4,2024-01-27,Vienna,,
\ No newline at end of file
diff --git a/dbrepo-metadata-service/rest-service/src/test/resources/init/weather.sql b/dbrepo-metadata-service/rest-service/src/test/resources/init/weather.sql
index 76f78ed34a0e64da912440d18ea61f3fef2595c7..b6dd8cff4cd1dabf0848697cb8db26db4e798601 100644
--- a/dbrepo-metadata-service/rest-service/src/test/resources/init/weather.sql
+++ b/dbrepo-metadata-service/rest-service/src/test/resources/init/weather.sql
@@ -118,7 +118,7 @@ VALUES ('2022-12-24 17:00:00', 10.0),
CREATE VIEW junit2 AS
(
-select `date`, `location`, `mintemp`, `rainfall`
+select `date`, `location` as loc, `location`, `mintemp`, `rainfall`
from `weather_aus`
where `location` = 'Albury');
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/listener/impl/MariadbListenerImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/listener/impl/MariadbListenerImpl.java
index 955cd10edd1b60cea7d429a56433aed1d505d376..d227a228bc9e544904a4c1bf238a0dbc219a583e 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/listener/impl/MariadbListenerImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/listener/impl/MariadbListenerImpl.java
@@ -40,12 +40,13 @@ public class MariadbListenerImpl implements DatabaseListener {
@Override
@Scheduled(fixedRateString = "${fda.obtainMetadataRate}", timeUnit = TimeUnit.SECONDS)
@Transactional
- public void updateStoredMetadata() throws QueryMalformedException, ColumnParseException,
- DatabaseNotFoundException, TableNotFoundException {
+ public void updateStoredMetadata() throws QueryMalformedException, ColumnParseException, DatabaseNotFoundException {
for (Long databaseId : databaseRepository.findAllOnlyIds()) {
try {
- databaseService.obtainMetadata(databaseId);
- } catch (DatabaseUnchangedException e) {
+ databaseService.obtainTablesMetadata(databaseId);
+ databaseService.obtainConstraints(databaseId);
+ databaseService.obtainViewsMetadata(databaseId);
+ } catch (DatabaseUnchangedException | TableMalformedException e) {
/* ignore */
}
}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java
index 59c0743a2e37a094f0107874755b3c78013163ba..e8045355d30e176c7aff17ce5f5cbae8bce51733 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java
@@ -114,7 +114,31 @@ public interface DatabaseService {
Database modifyImage(Long databaseId, byte[] image) throws DatabaseNotFoundException;
/**
- * Obtain metadata from database with given id to read table and view information (schema) and write it to the metadata database for management by DBRepo.
+ * Obtain table schema constraints for a database by given id.
+ *
+ * @param databaseId The database id.
+ * @return The updated database.
+ * @throws DatabaseNotFoundException The database was not found in the metadata database.
+ * @throws QueryMalformedException The inspect query (table/view) is malformed and has syntax issues.
+ * @throws TableMalformedException The table constraints are malformed.
+ */
+ Database obtainConstraints(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException, TableMalformedException;
+
+ /**
+ * Obtain metadata from database with given id to read table information (schema) and write it to the metadata database for management by DBRepo.
+ *
+ * @param databaseId The database id.
+ * @return The updated database.
+ * @throws DatabaseNotFoundException The database was not found in the metadata database.
+ * @throws QueryMalformedException The inspect query (table/view) is malformed and has syntax issues.
+ * @throws DatabaseUnchangedException The metadata database is up-to-date and knows about all tables/views in the data database(s).
+ * @throws ColumnParseException The columns could not be automatically parsed from the views.
+ */
+ Database obtainTablesMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException,
+ DatabaseUnchangedException, ColumnParseException;
+
+ /**
+ * Obtain metadata from database with given id to read view information (schema) and write it to the metadata database for management by DBRepo.
*
* @param databaseId The database id.
* @return The updated database.
@@ -123,6 +147,6 @@ public interface DatabaseService {
* @throws DatabaseUnchangedException The metadata database is up-to-date and knows about all tables/views in the data database(s).
* @throws ColumnParseException The columns could not be automatically parsed from the views.
*/
- Database obtainMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException,
- DatabaseUnchangedException, ColumnParseException, TableNotFoundException;
+ Database obtainViewsMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException,
+ DatabaseUnchangedException, ColumnParseException;
}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java
index d6350915820967217193eeceffae6a359a16484b..4d1f4679f71aa9212dd4bfadb9cf7e1a5597fef5 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/QueryService.java
@@ -89,6 +89,7 @@ public interface QueryService {
/**
* Select all data known in the database-table id tuple at a given time and return a page of specific size, using
* Instant to better abstract time concept (JDK 8) from SQL. We use the "mariadb" user for this.
+ * Precondition: page and size is not null
*
* @param databaseId The database id.
* @param tableId The table id.
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableColumnService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableColumnService.java
new file mode 100644
index 0000000000000000000000000000000000000000..5b486d986923e3cbe752d42bbee3f8326640c360
--- /dev/null
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableColumnService.java
@@ -0,0 +1,60 @@
+package at.tuwien.service;
+
+import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto;
+import at.tuwien.entities.database.Database;
+import at.tuwien.entities.database.table.Table;
+import at.tuwien.entities.database.table.columns.TableColumn;
+import at.tuwien.exception.DatabaseNotFoundException;
+import at.tuwien.exception.TableMalformedException;
+import at.tuwien.exception.TableNotFoundException;
+import org.springframework.transaction.annotation.Transactional;
+
+public interface TableColumnService {
+
+ /**
+ * Updates a table column
+ *
+ * @param databaseId The database id.
+ * @param tableId The table id.
+ * @param columnId The column id.
+ * @param updateDto The update data containing unit and concept uris.
+ * @return The updated table column, if successful.
+ * @throws TableNotFoundException The table was not found in the metadata database.
+ * @throws DatabaseNotFoundException The database was not found in the metadata database.
+ * @throws TableMalformedException The table seems malformed by the mapper.
+ * @throws TableNotFoundException The table is not found.
+ */
+ TableColumn update(Long databaseId, Long tableId, Long columnId, ColumnSemanticsUpdateDto updateDto)
+ throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException;
+
+ /**
+ * Finds a column in a given table with column id
+ *
+ * @param table The table.
+ * @param columnId The column id.
+ * @return The column, if successful.
+ * @throws TableMalformedException The requested column was not found in the table.
+ */
+ TableColumn findColumn(Table table, Long columnId) throws TableMalformedException;
+
+ /**
+ * Finds a column in a given table with column name.
+ *
+ * @param table The table.
+ * @param name The column name.
+ * @return The column, if successful.
+ * @throws TableMalformedException The requested column was not found in the table.
+ */
+ TableColumn findColumn(Table table, String name) throws TableMalformedException;
+
+ /**
+ * Finds a column in a database with given table name and given column name.
+ *
+ * @param database The database.
+ * @param tableName The table name.
+ * @param columnName The column name.
+ * @return The column, if successful.
+ * @throws TableMalformedException The requested column was not found in the database.
+ */
+ TableColumn findColumn(Database database, String tableName, String columnName) throws TableMalformedException;
+}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableService.java
index e1e57273a72923e0455635dd4cb53b525dcff08e..09f8012670437c33cc8ab0180a9cb74ea1361259 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableService.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/TableService.java
@@ -84,24 +84,6 @@ public interface TableService {
throws ImageNotSupportedException, DatabaseNotFoundException, TableMalformedException,
TableNameExistsException, QueryMalformedException, TableNotFoundException;
-
- /**
- * Updates a table column
- *
- * @param databaseId The database id.
- * @param tableId The table id.
- * @param columnId The column id.
- * @param updateDto The update data containing unit and concept uris.
- * @return The updated table column, if successful.
- * @throws TableNotFoundException The table was not found in the metadata database.
- * @throws DatabaseNotFoundException The database was not found in the metadata database.
- * @throws TableMalformedException The table seems malformed by the mapper.
- * @throws TableNotFoundException The table is not found.
- */
- TableColumn update(Long databaseId, Long tableId, Long columnId, ColumnSemanticsUpdateDto updateDto,
- String authorization) throws TableNotFoundException, DatabaseNotFoundException,
- TableMalformedException;
-
/**
* Deletes a table from the database in the metadata database and data database.
*
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java
index 59ef1aa88e5fe765e28c135458e1ac2a30ea70b2..183f05711653ca47179a212a2797b05bc1059e38 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/MariaDbServiceImpl.java
@@ -9,8 +9,11 @@ import at.tuwien.entities.container.image.ContainerImageDate;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.View;
import at.tuwien.entities.database.table.Table;
+import at.tuwien.entities.database.table.columns.TableColumn;
import at.tuwien.entities.database.table.constraints.Constraints;
import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKey;
+import at.tuwien.entities.database.table.constraints.foreignKey.ForeignKeyReference;
+import at.tuwien.entities.database.table.constraints.foreignKey.ReferenceType;
import at.tuwien.entities.database.table.constraints.unique.Unique;
import at.tuwien.entities.user.User;
import at.tuwien.exception.*;
@@ -21,9 +24,7 @@ import at.tuwien.mapper.ViewMapper;
import at.tuwien.repository.mdb.ContainerRepository;
import at.tuwien.repository.mdb.DatabaseRepository;
import at.tuwien.repository.sdb.DatabaseIdxRepository;
-import at.tuwien.service.ContainerService;
-import at.tuwien.service.DatabaseService;
-import at.tuwien.service.UserService;
+import at.tuwien.service.*;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import lombok.extern.log4j.Log4j2;
import net.sf.jsqlparser.JSQLParserException;
@@ -50,6 +51,7 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
private final DatabaseMapper databaseMapper;
private final ContainerService containerService;
private final DatabaseRepository databaseRepository;
+ private final TableColumnService tableColumnService;
private final ContainerRepository containerRepository;
private final DatabaseIdxRepository databaseIdxRepository;
@@ -57,7 +59,8 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
public MariaDbServiceImpl(ViewMapper viewMapper, QueryConfig queryConfig, QueryMapper queryMapper,
TableMapper tableMapper, UserService userService, DatabaseMapper databaseMapper,
ContainerService containerService, DatabaseRepository databaseRepository,
- ContainerRepository containerRepository, DatabaseIdxRepository databaseIdxRepository) {
+ TableColumnService tableColumnService, ContainerRepository containerRepository,
+ DatabaseIdxRepository databaseIdxRepository) {
this.viewMapper = viewMapper;
this.queryConfig = queryConfig;
this.queryMapper = queryMapper;
@@ -66,6 +69,7 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
this.databaseMapper = databaseMapper;
this.containerService = containerService;
this.databaseRepository = databaseRepository;
+ this.tableColumnService = tableColumnService;
this.containerRepository = containerRepository;
this.databaseIdxRepository = databaseIdxRepository;
}
@@ -232,13 +236,47 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
@Override
@Transactional
- public Database obtainMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException,
+ public Database obtainConstraints(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException,
+ TableMalformedException {
+ /* check */
+ final Database database = findById(databaseId);
+ final List<Table> diffTables = database.getTables()
+ .stream()
+ .filter(t -> !t.getProcessedConstraints())
+ .toList();
+ /* obtain constraints */
+ log.info("Database with id {} contains {} table(s) with unknown constraint(s)", databaseId, diffTables.size());
+ final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(), database.getContainer(), database);
+ try {
+ final Connection connection = dataSource.getConnection();
+ for (Table table : diffTables) {
+ final PreparedStatement preparedStatement = queryMapper.databaseToDatabaseConstraintMetadata(connection, table.getDatabase().getInternalName(), table.getInternalName());
+ final Constraints constraints = resultSetTableToObtainedConstraintsMetadata(databaseId, table, preparedStatement.executeQuery());
+ table.setConstraints(constraints);
+ table.setProcessedConstraints(true);
+ }
+ } catch (SQLException e) {
+ log.error("Failed to obtain constraint information in database with id {}: {}", database.getId(), e.getMessage());
+ throw new QueryMalformedException("Failed to obtain constraint information in database with id " + database.getId() + ": " + e.getMessage(), e);
+ } finally {
+ dataSource.close();
+ }
+ /* update in metadata database */
+ final Database entity = databaseRepository.save(database);
+ /* save in open search database */
+ databaseIdxRepository.save(databaseMapper.databaseToDatabaseDto(entity));
+ log.info("Updated database with id {} in metadata database & search database", entity.getId());
+ return entity;
+ }
+
+ @Override
+ @Transactional
+ public Database obtainTablesMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException,
ColumnParseException {
/* check */
final Database database = findById(databaseId);
final List<Table> diffTables;
final List<Table> knownTables;
- final List<View> diffViews;
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(), database.getContainer(), database);
try {
final Connection connection = dataSource.getConnection();
@@ -268,12 +306,6 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
return obtainedTable;
})
.toList();
- final List<View> views = tableMapper.resultListToViewList(preparedStatement0.executeQuery(), database);
- diffViews = views.stream()
- .filter(view -> database.getViews()
- .stream()
- .noneMatch(v -> v.getInternalName().equals(view.getInternalName())))
- .toList();
/* default times */
final Optional<ContainerImageDate> defaultDateFormat = containerRepository.findDefaultDateFormat();
if (defaultDateFormat.isEmpty()) {
@@ -286,7 +318,7 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
throw new ColumnParseException("Failed to find default timestamp format in metadata database");
}
/* obtain table schema */
- log.info("Database with id {} contains {} unknown table(s) and {} unknown view(s)", databaseId, diffTables.size(), diffViews.size());
+ log.info("Database with id {} contains {} unknown table(s)", databaseId, diffTables.size());
log.debug("database with id {} misses table(s) in metadata database: {}", databaseId, diffTables.stream().map(Table::getInternalName).toList());
database.getTables().replaceAll(table -> {
final Optional<Table> optional = knownTables.stream()
@@ -308,8 +340,7 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
preparedStatement2.execute();
log.info("Enabled system-versioning for table with name {}", table.getInternalName());
}
- final PreparedStatement preparedStatement2 = queryMapper.databaseToDatabaseConstraintMetadata(connection, table.getDatabase().getInternalName(), table.getInternalName());
- table.setConstraints(resultSetTableToObtainedConstraintsMetadata(preparedStatement2.executeQuery()));
+ table.setProcessedConstraints(false);
final PreparedStatement preparedStatement3 = tableMapper.tableToCreateHistoryViewRawQuery(connection, table);
preparedStatement3.executeUpdate();
database.getTables().add(table);
@@ -320,6 +351,50 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
} finally {
dataSource.close();
}
+ /* update in metadata database */
+ final Database entity = databaseRepository.save(database);
+ /* save in open search database */
+ databaseIdxRepository.save(databaseMapper.databaseToDatabaseDto(entity));
+ log.info("Updated database with id {} in metadata database & search database", entity.getId());
+ return entity;
+ }
+
+ @Override
+ @Transactional
+ public Database obtainViewsMetadata(Long databaseId) throws DatabaseNotFoundException, QueryMalformedException,
+ ColumnParseException {
+ /* check */
+ final Database database = findById(databaseId);
+ final List<View> diffViews;
+ final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(), database.getContainer(), database);
+ try {
+ final Connection connection = dataSource.getConnection();
+ final PreparedStatement preparedStatement0 = databaseMapper.databaseToDatabaseMetadata(connection, database);
+ final List<View> views = tableMapper.resultListToViewList(preparedStatement0.executeQuery(), database);
+ diffViews = views.stream()
+ .filter(view -> database.getViews()
+ .stream()
+ .noneMatch(v -> v.getInternalName().equals(view.getInternalName())))
+ .toList();
+ /* obtain table schema */
+ log.info("Database with id {} contains {} unknown view(s)", databaseId, diffViews.size());
+ /* default times */
+ final Optional<ContainerImageDate> defaultDateFormat = containerRepository.findDefaultDateFormat();
+ if (defaultDateFormat.isEmpty()) {
+ log.error("Failed to find default date format in metadata database");
+ throw new ColumnParseException("Failed to find default date format in metadata database");
+ }
+ final Optional<ContainerImageDate> defaultTimestampFormat = containerRepository.findDefaultTimestampFormat();
+ if (defaultTimestampFormat.isEmpty()) {
+ log.error("Failed to find default timestamp format in metadata database");
+ throw new ColumnParseException("Failed to find default timestamp format in metadata database");
+ }
+ } catch (SQLException e) {
+ log.error("Failed to obtain schema information in database with id {}: {}", database.getId(), e.getMessage());
+ throw new QueryMalformedException("Failed to obtain schema information in database with id " + database.getId() + ": " + e.getMessage(), e);
+ } finally {
+ dataSource.close();
+ }
/* obtain view schema */
log.debug("database with id {} misses view(s) in metadata database: {}", databaseId, diffViews.stream().map(View::getInternalName).toList());
for (View view : diffViews) {
@@ -344,16 +419,75 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
return entity;
}
- @Transactional
- public Constraints resultSetTableToObtainedConstraintsMetadata(ResultSet resultSet) throws SQLException {
+ @Transactional(readOnly = true)
+ public Constraints resultSetTableToObtainedConstraintsMetadata(Long databaseId, Table table, ResultSet resultSet)
+ throws SQLException, DatabaseNotFoundException, TableMalformedException {
+ final Database database = find(databaseId);
final Set<String> checks = new LinkedHashSet<>();
final List<Unique> uniques = new LinkedList<>();
final List<ForeignKey> foreignKeys = new LinkedList<>();
while (resultSet.next()) {
if (resultSet.getString(1).equals("CHECK")) {
+ /* check constraints */
checks.add(resultSet.getString(4));
} else if (resultSet.getString(1).equals("FOREIGN KEY")) {
- // TODO
+ /* foreign key constraints */
+ final List<ForeignKeyReference> foreignKeyReferences = new LinkedList<>();
+ final String foreignKeyName = resultSet.getString(2);
+ if (foreignKeys.stream().anyMatch(fk -> fk.getName().equals(foreignKeyName))) {
+ final Optional<ForeignKey> optional = foreignKeys.stream()
+ .filter(fk -> fk.getName().equals(foreignKeyName))
+ .findFirst();
+ if (optional.isEmpty()) {
+ /* should never happen */
+ continue;
+ }
+ final ForeignKey foreignKey = optional.get();
+ foreignKey.getReferences()
+ .add(queryMapper.foreignKeyToForeignKeyReference(foreignKey,
+ tableColumnService.findColumn(database, resultSet.getString(6), resultSet.getString(8)),
+ tableColumnService.findColumn(table, resultSet.getString(7))));
+ }
+ final ForeignKey foreignKey;
+ try {
+ foreignKey = ForeignKey.builder()
+ .name(foreignKeyName)
+ .table(table)
+ .referencedTable(find(database, resultSet.getString(6)))
+ .references(foreignKeyReferences)
+ .onDelete(ReferenceType.NO_ACTION)
+ .onUpdate(ReferenceType.NO_ACTION)
+ .build();
+ } catch (TableNotFoundException e) {
+ /* ignore */
+ return null;
+ }
+ final ForeignKeyReference fk = ForeignKeyReference.builder()
+ .foreignKey(foreignKey)
+ .column(tableColumnService.findColumn(table, resultSet.getString(7)))
+ .referencedColumn(tableColumnService.findColumn(database, resultSet.getString(6), resultSet.getString(8)))
+ .build();
+ foreignKey.setReferences(List.of(fk));
+ foreignKeys.add(foreignKey);
+ } else if (resultSet.getString(1).equals("UNIQUE")) {
+ /* unique constraints */
+ final String uniqueConstraintName = resultSet.getString(1);
+ final Optional<Unique> optional = uniques.stream().filter(u -> u.getName().equals(uniqueConstraintName)).findFirst();
+ if (optional.isPresent()) {
+ log.debug("unique constraint {} already present: add column", uniqueConstraintName);
+ optional.get()
+ .getColumns()
+ .add(tableColumnService.findColumn(table, resultSet.getString(7)));
+ continue;
+ }
+ final List<TableColumn> columns = new LinkedList<>();
+ columns.add(tableColumnService.findColumn(table, resultSet.getString(7)));
+ final Unique uk = Unique.builder()
+ .name(uniqueConstraintName)
+ .table(table)
+ .columns(columns)
+ .build();
+ uniques.add(uk);
}
}
final Constraints constraints = Constraints.builder()
@@ -367,4 +501,16 @@ public class MariaDbServiceImpl extends HibernateConnector implements DatabaseSe
return constraints;
}
+ public Table find(Database database, String internalName) throws DatabaseNotFoundException, TableNotFoundException {
+ final Optional<Table> table = database.getTables()
+ .stream()
+ .filter(t -> t.getInternalName().equals(internalName))
+ .findFirst();
+ if (table.isEmpty()) {
+ log.error("Failed to find table with internal name {} in metadata database", internalName);
+ throw new TableNotFoundException("Failed to find table with internal name " + internalName + " in metadata database");
+ }
+ return table.get();
+ }
+
}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
index 2707d3d3579b33b220d3f94a900942e80fd6f17a..4b1d5bb95291a4a825e7513d88fbdf23794315e8 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/QueryServiceImpl.java
@@ -27,10 +27,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.security.Principal;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
+import java.sql.*;
import java.time.Instant;
import java.time.format.DateTimeParseException;
import java.util.List;
@@ -120,7 +117,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
return executeCountNonPersistent(databaseId, statement);
}
- private PreparedStatement prepareStatement(Connection connection, String statement) throws QueryMalformedException {
+ public PreparedStatement prepareStatement(Connection connection, String statement) throws QueryMalformedException {
try {
return connection.prepareStatement(statement);
} catch (SQLException e) {
@@ -129,7 +126,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
}
}
- private QueryResultDto executeNonPersistent(Long databaseId, String statement, List<TableColumn> columns)
+ public QueryResultDto executeNonPersistent(Long databaseId, String statement, List<TableColumn> columns)
throws QueryMalformedException, DatabaseNotFoundException, TableMalformedException {
/* find */
final Database database = databaseService.find(databaseId);
@@ -150,7 +147,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
}
}
- private Long executeCountNonPersistent(Long databaseId, String statement)
+ public Long executeCountNonPersistent(Long databaseId, String statement)
throws QueryMalformedException, TableMalformedException, DatabaseNotFoundException, QueryStoreException {
/* find */
final Database database = databaseService.find(databaseId);
@@ -253,7 +250,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
return retrieveBlobAsResource(database.getContainer(), filename);
}
- private ExportResource retrieveBlobAsResource(Container container, String filename) throws DataDbSidecarException,
+ public ExportResource retrieveBlobAsResource(Container container, String filename) throws DataDbSidecarException,
FileStorageException, DataProcessingException {
/* upload from sidecar into blob storage */
dataDbSidecarGateway.exportFile(container.getSidecarHost(), container.getSidecarPort(), filename);
@@ -331,7 +328,7 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
final Database database = databaseService.find(databaseId);
final Table table = tableService.find(databaseId, tableId);
/* run query */
- if (data.getKeys().size() == 0) return;
+ if (data.getKeys().isEmpty()) return;
final ComboPooledDataSource dataSource = getPrivilegedDataSource(database.getContainer().getImage(),
database.getContainer(), database);
/* prepare the statement */
@@ -362,8 +359,12 @@ public class QueryServiceImpl extends HibernateConnector implements QueryService
database.getContainer(), database);
try {
final Connection connection = dataSource.getConnection();
- queryMapper.importCsvQuery(connection, table, data);
+ final PreparedStatement statement = queryMapper.pathToRawInsertQuery(connection, table, data);
+ statement.executeUpdate();
} catch (SQLException e) {
+ log.error("Failed to open connection to data database: {}", e.getMessage());
+ throw new TableMalformedException("Failed to open connection to data database: " + e.getMessage(), e);
+ } catch (QueryMalformedException e) {
log.error("Failed to import csv: {}", e.getMessage());
throw new TableMalformedException("Failed to import csv: " + e.getMessage(), e);
} finally {
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableColumnServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableColumnServiceImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..46763db156bd332ed2e3385cc473230c94dd0ccf
--- /dev/null
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableColumnServiceImpl.java
@@ -0,0 +1,153 @@
+package at.tuwien.service.impl;
+
+import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto;
+import at.tuwien.entities.database.Database;
+import at.tuwien.entities.database.table.Table;
+import at.tuwien.entities.database.table.columns.TableColumn;
+import at.tuwien.entities.database.table.columns.TableColumnConcept;
+import at.tuwien.entities.database.table.columns.TableColumnUnit;
+import at.tuwien.exception.*;
+import at.tuwien.mapper.DatabaseMapper;
+import at.tuwien.repository.mdb.DatabaseRepository;
+import at.tuwien.repository.sdb.DatabaseIdxRepository;
+import at.tuwien.service.SemanticService;
+import at.tuwien.service.TableColumnService;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Stream;
+
+@Log4j2
+@Service
+public class TableColumnServiceImpl implements TableColumnService {
+
+ private final DatabaseMapper databaseMapper;
+ private final SemanticService semanticService;
+ private final DatabaseRepository databaseRepository;
+ private final DatabaseIdxRepository databaseIdxRepository;
+
+ @Autowired
+ public TableColumnServiceImpl(DatabaseMapper databaseMapper, SemanticService semanticService,
+ DatabaseRepository databaseRepository, DatabaseIdxRepository databaseIdxRepository) {
+ this.databaseMapper = databaseMapper;
+ this.semanticService = semanticService;
+ this.databaseRepository = databaseRepository;
+ this.databaseIdxRepository = databaseIdxRepository;
+ }
+
+ @Transactional(readOnly = true)
+ public Database find(Long databaseId) throws DatabaseNotFoundException {
+ final Optional<Database> database = databaseRepository.findById(databaseId);
+ if (database.isEmpty()) {
+ log.error("Failed to find database with id {} in metadata database", databaseId);
+ throw new DatabaseNotFoundException("could not find database with id " + databaseId + " in metadata database");
+ }
+ return database.get();
+ }
+
+ @Transactional(readOnly = true)
+ public Table find(Long databaseId, Long tableId) throws DatabaseNotFoundException, TableNotFoundException {
+ final Optional<Table> table = find(databaseId)
+ .getTables()
+ .stream()
+ .filter(t -> t.getId().equals(tableId))
+ .findFirst();
+ if (table.isEmpty()) {
+ log.error("Failed to find table with id {} in metadata database", tableId);
+ throw new TableNotFoundException("Failed to find table with id " + tableId + " in metadata database");
+ }
+ return table.get();
+ }
+
+ @Override
+ @Transactional
+ public TableColumn update(Long databaseId, Long tableId, Long columnId, ColumnSemanticsUpdateDto updateDto)
+ throws TableNotFoundException, DatabaseNotFoundException, TableMalformedException {
+ final Table table = find(databaseId, tableId);
+ final TableColumn column = findColumn(table, columnId);
+ /* assign */
+ if (updateDto.getUnitUri() != null) {
+ try {
+ column.setUnit(semanticService.findUnit(updateDto.getUnitUri()));
+ log.debug("found unit with uri {} in metadata database", updateDto.getUnitUri());
+ } catch (UnitNotFoundException e) {
+ final TableColumnUnit unit = TableColumnUnit.builder()
+ .uri(updateDto.getUnitUri())
+ .build();
+ column.setUnit(unit);
+ }
+ } else {
+ column.setUnit(null);
+ }
+ if (updateDto.getConceptUri() != null) {
+ try {
+ column.setConcept(semanticService.findConcept(updateDto.getConceptUri()));
+ log.debug("found concept with uri {} in metadata database", updateDto.getConceptUri());
+ } catch (ConceptNotFoundException e) {
+ final TableColumnConcept concept = TableColumnConcept.builder()
+ .uri(updateDto.getConceptUri())
+ .build();
+ column.setConcept(concept);
+ }
+ } else {
+ column.setConcept(null);
+ }
+ /* update in metadata database */
+ table.getColumns().set(table.getColumns().indexOf(column), column);
+ databaseRepository.save(table.getDatabase());
+ /* update in open search database */
+ databaseIdxRepository.save(databaseMapper.databaseToDatabaseDto(find(databaseId)));
+ log.info("Updated table column with id {} of table with id {} in metadata database & search database", columnId, tableId);
+ return column;
+ }
+
+ @Override
+ @Transactional(readOnly = true)
+ public TableColumn findColumn(Table table, Long columnId) throws TableMalformedException {
+ final Optional<TableColumn> optional = table.getColumns()
+ .stream()
+ .filter(c -> c.getId().equals(columnId))
+ .findFirst();
+ if (optional.isEmpty()) {
+ log.error("Failed to find column with id {} in metadata database", columnId);
+ throw new TableMalformedException("Failed to find column with id " + columnId + " in metadata database");
+ }
+ return optional.get();
+ }
+
+ @Override
+ @Transactional(readOnly = true)
+ public TableColumn findColumn(Table table, String name) throws TableMalformedException {
+ final Optional<TableColumn> optional = table.getColumns()
+ .stream()
+ .filter(c -> c.getInternalName().equals(name))
+ .findFirst();
+ if (optional.isEmpty()) {
+ log.error("Failed to find column with name {} in table with name {}", name, table.getInternalName());
+ throw new TableMalformedException("Failed to find column with name " + name + " in table with name " + table.getInternalName());
+ }
+ return optional.get();
+ }
+
+ @Override
+ @Transactional(readOnly = true)
+ public TableColumn findColumn(Database database, String tableName, String columnName)
+ throws TableMalformedException {
+ final Optional<TableColumn> optional = database.getTables()
+ .stream()
+ .filter(t -> t.getInternalName().equals(tableName))
+ .map(Table::getColumns)
+ .flatMap(List::stream)
+ .filter(c -> c.getInternalName().equals(columnName))
+ .findFirst();
+ if (optional.isEmpty()) {
+ log.error("Failed to find column {}.{} in database with id {}", tableName, columnName, database.getId());
+ throw new TableMalformedException("Failed to find column " + tableName + "." + columnName + " in database with id " + database.getId());
+ }
+ return optional.get();
+ }
+}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
index 3201ad9ea0e7b8ea839da97ed6a0b67352309e4d..525383b115cdfc36806dac9677adc076715d955c 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
@@ -40,19 +40,17 @@ public class TableServiceImpl extends HibernateConnector implements TableService
private final TableMapper tableMapper;
private final DatabaseMapper databaseMapper;
private final DatabaseService databaseService;
- private final SemanticService semanticService;
private final DatabaseRepository databaseRepository;
private final DatabaseIdxRepository databaseIdxRepository;
@Autowired
public TableServiceImpl(QueryMapper queryMapper, TableMapper tableMapper, DatabaseMapper databaseMapper,
- DatabaseService databaseService, SemanticService semanticService,
- DatabaseRepository databaseRepository, DatabaseIdxRepository databaseIdxRepository) {
+ DatabaseService databaseService, DatabaseRepository databaseRepository,
+ DatabaseIdxRepository databaseIdxRepository) {
this.queryMapper = queryMapper;
this.tableMapper = tableMapper;
this.databaseMapper = databaseMapper;
this.databaseService = databaseService;
- this.semanticService = semanticService;
this.databaseRepository = databaseRepository;
this.databaseIdxRepository = databaseIdxRepository;
}
@@ -207,49 +205,6 @@ public class TableServiceImpl extends HibernateConnector implements TableService
return optionalEntity.get();
}
- @Override
- @Transactional
- public TableColumn update(Long databaseId, Long tableId, Long columnId, ColumnSemanticsUpdateDto updateDto,
- String authorization) throws TableNotFoundException, DatabaseNotFoundException,
- TableMalformedException {
- final Table table = find(databaseId, tableId);
- final TableColumn column = findColumn(table, columnId);
- /* assign */
- if (updateDto.getUnitUri() != null) {
- try {
- column.setUnit(semanticService.findUnit(updateDto.getUnitUri()));
- log.debug("found unit with uri {} in metadata database", updateDto.getUnitUri());
- } catch (UnitNotFoundException e) {
- final TableColumnUnit unit = TableColumnUnit.builder()
- .uri(updateDto.getUnitUri())
- .build();
- column.setUnit(unit);
- }
- } else {
- column.setUnit(null);
- }
- if (updateDto.getConceptUri() != null) {
- try {
- column.setConcept(semanticService.findConcept(updateDto.getConceptUri()));
- log.debug("found concept with uri {} in metadata database", updateDto.getConceptUri());
- } catch (ConceptNotFoundException e) {
- final TableColumnConcept concept = TableColumnConcept.builder()
- .uri(updateDto.getConceptUri())
- .build();
- column.setConcept(concept);
- }
- } else {
- column.setConcept(null);
- }
- /* update in metadata database */
- table.getColumns().set(table.getColumns().indexOf(column), column);
- databaseRepository.save(table.getDatabase());
- /* update in open search database */
- databaseIdxRepository.save(databaseMapper.databaseToDatabaseDto(databaseService.find(databaseId)));
- log.info("Updated table column with id {} of table with id {} in metadata database & search database", columnId, tableId);
- return column;
- }
-
@Override
@Transactional
public void deleteTable(Long databaseId, Long tableId)
@@ -278,24 +233,4 @@ public class TableServiceImpl extends HibernateConnector implements TableService
log.info("Deleted table with id {} in open search database", table.getId());
}
- /**
- * Finds a column in a given table with column id
- *
- * @param table The table.
- * @param columnId The column id.
- * @return The column, if successful.
- * @throws TableMalformedException The requested column was not found in the table.
- */
- protected TableColumn findColumn(Table table, Long columnId) throws TableMalformedException {
- final Optional<TableColumn> optional = table.getColumns()
- .stream()
- .filter(c -> c.getId().equals(columnId))
- .findFirst();
- if (optional.isEmpty()) {
- log.error("Failed to find column with id {} in metadata database", columnId);
- throw new TableMalformedException("Failed to find column with id " + columnId + " in metadata database");
- }
- return optional.get();
- }
-
}
diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java
index 5fadde3b28d8530c3c545763957daf1091756697..d9dc02f88a602cde25013bb3feb0f67b78ddcb19 100644
--- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java
+++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/ViewServiceImpl.java
@@ -147,7 +147,7 @@ public class ViewServiceImpl extends HibernateConnector implements ViewService {
columns = queryMapper.parseColumns(data.getQuery(), database);
} catch (JSQLParserException e) {
log.error("Failed to map/parse columns: {}", e.getMessage());
- throw new QueryMalformedException(e.getMessage(), e);
+ throw new QueryMalformedException("Failed to map/parse columns: " + e.getMessage(), e);
}
try {
final Connection connection = dataSource.getConnection();
diff --git a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java
index 6d60d4219b4223816e76a9d452c8606d8c58cf05..8fc32dc4671ca372cb5d397b04d5e86c8dcf2727 100644
--- a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java
+++ b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java
@@ -1334,6 +1334,7 @@ public abstract class BaseTest {
public final static String TABLE_1_NAME = "Weather AUS";
public final static String TABLE_1_INTERNALNAME = "weather_aus";
public final static Boolean TABLE_1_VERSIONED = true;
+ public final static Boolean TABLE_1_PROCESSED_CONSTRAINTS = true;
public final static String TABLE_1_DESCRIPTION = "Weather in the world";
public final static String TABLE_1_QUEUE_NAME = TABLE_1_INTERNALNAME;
public final static String TABLE_1_ROUTING_KEY = "dbrepo\\." + DATABASE_1_EXCHANGE + "\\." + TABLE_1_QUEUE_NAME;
@@ -1348,6 +1349,7 @@ public abstract class BaseTest {
.created(TABLE_1_CREATED)
.internalName(TABLE_1_INTERNALNAME)
.isVersioned(TABLE_1_VERSIONED)
+ .processedConstraints(TABLE_1_PROCESSED_CONSTRAINTS)
.description(TABLE_1_DESCRIPTION)
.name(TABLE_1_NAME)
.queueName(TABLE_1_QUEUE_NAME)
@@ -1393,6 +1395,7 @@ public abstract class BaseTest {
public final static String TABLE_2_NAME = "Weather Location";
public final static String TABLE_2_INTERNALNAME = "weather_location";
public final static Boolean TABLE_2_VERSIONED = true;
+ public final static Boolean TABLE_2_PROCESSED_CONSTRAINTS = true;
public final static String TABLE_2_DESCRIPTION = "Weather location";
public final static String TABLE_2_QUEUE_NAME = TABLE_2_INTERNALNAME;
public final static String TABLE_2_ROUTING_KEY = "dbrepo\\." + DATABASE_1_EXCHANGE + "\\." + TABLE_2_QUEUE_NAME;
@@ -1406,6 +1409,7 @@ public abstract class BaseTest {
.created(TABLE_2_CREATED)
.internalName(TABLE_2_INTERNALNAME)
.isVersioned(TABLE_2_VERSIONED)
+ .processedConstraints(TABLE_2_PROCESSED_CONSTRAINTS)
.description(TABLE_2_DESCRIPTION)
.name(TABLE_2_NAME)
.lastModified(TABLE_2_LAST_MODIFIED)
@@ -1448,6 +1452,7 @@ public abstract class BaseTest {
public final static String TABLE_3_NAME = "Sensor";
public final static String TABLE_3_INTERNALNAME = "sensor";
public final static Boolean TABLE_3_VERSIONED = true;
+ public final static Boolean TABLE_3_PROCESSED_CONSTRAINTS = true;
public final static String TABLE_3_DESCRIPTION = "Some sensor data";
public final static String TABLE_3_QUEUE_NAME = TABLE_3_INTERNALNAME;
public final static String TABLE_3_ROUTING_KEY = "dbrepo\\." + DATABASE_1_EXCHANGE + "\\." + TABLE_3_QUEUE_NAME;
@@ -1461,6 +1466,7 @@ public abstract class BaseTest {
.created(TABLE_3_CREATED)
.internalName(TABLE_3_INTERNALNAME)
.isVersioned(TABLE_3_VERSIONED)
+ .processedConstraints(TABLE_3_PROCESSED_CONSTRAINTS)
.description(TABLE_3_DESCRIPTION)
.name(TABLE_3_NAME)
.lastModified(TABLE_3_LAST_MODIFIED)
@@ -1529,6 +1535,7 @@ public abstract class BaseTest {
public final static String TABLE_5_NAME = "zoo";
public final static String TABLE_5_INTERNALNAME = "zoo";
public final static Boolean TABLE_5_VERSIONED = true;
+ public final static Boolean TABLE_5_PROCESSED_CONSTRAINTS = true;
public final static String TABLE_5_DESCRIPTION = "Some Kaggle dataset";
public final static String TABLE_5_QUEUE_NAME = TABLE_5_INTERNALNAME;
public final static String TABLE_5_ROUTING_KEY = "dbrepo\\." + DATABASE_2_EXCHANGE + "\\." + TABLE_5_QUEUE_NAME;
@@ -1541,6 +1548,7 @@ public abstract class BaseTest {
.created(Instant.now())
.internalName(TABLE_5_INTERNALNAME)
.isVersioned(TABLE_5_VERSIONED)
+ .processedConstraints(TABLE_5_PROCESSED_CONSTRAINTS)
.description(TABLE_5_DESCRIPTION)
.name(TABLE_5_NAME)
.lastModified(TABLE_5_LAST_MODIFIED)
@@ -1579,6 +1587,7 @@ public abstract class BaseTest {
public final static String TABLE_6_NAME = "names";
public final static String TABLE_6_INTERNALNAME = "names";
public final static Boolean TABLE_6_VERSIONED = true;
+ public final static Boolean TABLE_6_PROCESSED_CONSTRAINTS = true;
public final static String TABLE_6_DESCRIPTION = "Some names dataset";
public final static String TABLE_6_QUEUE_NAME = TABLE_6_INTERNALNAME;
public final static String TABLE_6_ROUTING_KEY = "dbrepo\\." + DATABASE_2_EXCHANGE + "\\." + TABLE_6_QUEUE_NAME;
@@ -1591,6 +1600,7 @@ public abstract class BaseTest {
.created(TABLE_6_CREATED)
.internalName(TABLE_6_INTERNALNAME)
.isVersioned(TABLE_6_VERSIONED)
+ .processedConstraints(TABLE_6_PROCESSED_CONSTRAINTS)
.description(TABLE_6_DESCRIPTION)
.name(TABLE_6_NAME)
.lastModified(TABLE_6_LAST_MODIFIED)
@@ -1625,6 +1635,7 @@ public abstract class BaseTest {
public final static String TABLE_7_NAME = "likes";
public final static String TABLE_7_INTERNAL_NAME = "likes";
public final static Boolean TABLE_7_VERSIONED = true;
+ public final static Boolean TABLE_7_PROCESSED_CONSTRAINTS = true;
public final static String TABLE_7_DESCRIPTION = "Some likes dataset";
public final static String TABLE_7_QUEUE_NAME = TABLE_7_INTERNAL_NAME;
public final static String TABLE_7_ROUTING_KEY = "dbrepo\\." + DATABASE_2_EXCHANGE + "\\." + TABLE_7_QUEUE_NAME;
@@ -1637,6 +1648,7 @@ public abstract class BaseTest {
.created(TABLE_7_CREATED)
.internalName(TABLE_7_INTERNAL_NAME)
.isVersioned(TABLE_7_VERSIONED)
+ .processedConstraints(TABLE_7_PROCESSED_CONSTRAINTS)
.description(TABLE_7_DESCRIPTION)
.name(TABLE_7_NAME)
.lastModified(TABLE_7_LAST_MODIFIED)
@@ -1671,6 +1683,7 @@ public abstract class BaseTest {
public final static String TABLE_4_NAME = "Sensor";
public final static String TABLE_4_INTERNAL_NAME = "sensor";
public final static Boolean TABLE_4_VERSIONED = true;
+ public final static Boolean TABLE_4_PROCESSED_CONSTRAINTS = true;
public final static String TABLE_4_DESCRIPTION = "Hello sensor";
public final static String TABLE_4_QUEUE_NAME = TABLE_4_INTERNAL_NAME;
public final static String TABLE_4_ROUTING_KEY = "dbrepo\\." + DATABASE_1_EXCHANGE + "\\." + TABLE_4_QUEUE_NAME;
@@ -1688,6 +1701,7 @@ public abstract class BaseTest {
.routingKey(TABLE_4_ROUTING_KEY)
.columns(List.of() /* TABLE_4_COLUMNS */)
.isVersioned(TABLE_4_VERSIONED)
+ .processedConstraints(TABLE_4_PROCESSED_CONSTRAINTS)
.createdBy(USER_1_ID)
.ownedBy(USER_1_ID)
.owner(USER_1)
@@ -1772,6 +1786,7 @@ public abstract class BaseTest {
public final static String TABLE_8_NAME = "mfcc";
public final static String TABLE_8_INTERNAL_NAME = "mfcc";
public final static Boolean TABLE_8_VERSIONED = true;
+ public final static Boolean TABLE_8_PROCESSED_CONSTRAINTS = true;
public final static String TABLE_8_DESCRIPTION = "Hello mfcc";
public final static String TABLE_8_QUEUE_NAME = TABLE_8_INTERNAL_NAME;
public final static String TABLE_8_ROUTING_KEY = "dbrepo\\." + DATABASE_3_EXCHANGE + "\\." + TABLE_8_QUEUE_NAME;
@@ -1784,6 +1799,7 @@ public abstract class BaseTest {
.internalName(TABLE_8_INTERNAL_NAME)
.description(TABLE_8_DESCRIPTION)
.isVersioned(TABLE_8_VERSIONED)
+ .processedConstraints(TABLE_8_PROCESSED_CONSTRAINTS)
.database(null /* DATABASE_1 */)
.name(TABLE_8_NAME)
.queueName(TABLE_8_QUEUE_NAME)
@@ -2039,7 +2055,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_1_TYPE_DTO = ColumnTypeDto.BIGINT;
public final static Long COLUMN_4_1_DATE_FORMAT = null;
public final static Boolean COLUMN_4_1_NULL = false;
- public final static Boolean COLUMN_4_1_UNIQUE = true;
public final static Boolean COLUMN_4_1_AUTO_GENERATED = true;
public final static String COLUMN_4_1_FOREIGN_KEY = null;
public final static String COLUMN_4_1_CHECK = null;
@@ -2059,7 +2074,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_2_TYPE_DTO = ColumnTypeDto.VARCHAR;
public final static Long COLUMN_4_2_DATE_FORMAT = null;
public final static Boolean COLUMN_4_2_NULL = true;
- public final static Boolean COLUMN_4_2_UNIQUE = false;
public final static Boolean COLUMN_4_2_AUTO_GENERATED = false;
public final static String COLUMN_4_2_FOREIGN_KEY = null;
public final static String COLUMN_4_2_CHECK = null;
@@ -2079,7 +2093,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_3_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_3_DATE_FORMAT = null;
public final static Boolean COLUMN_4_3_NULL = true;
- public final static Boolean COLUMN_4_3_UNIQUE = false;
public final static Boolean COLUMN_4_3_AUTO_GENERATED = false;
public final static String COLUMN_4_3_FOREIGN_KEY = null;
public final static String COLUMN_4_3_CHECK = null;
@@ -2099,7 +2112,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_4_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_4_DATE_FORMAT = null;
public final static Boolean COLUMN_4_4_NULL = true;
- public final static Boolean COLUMN_4_4_UNIQUE = false;
public final static Boolean COLUMN_4_4_AUTO_GENERATED = false;
public final static String COLUMN_4_4_FOREIGN_KEY = null;
public final static String COLUMN_4_4_CHECK = null;
@@ -2119,7 +2131,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_5_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_5_DATE_FORMAT = null;
public final static Boolean COLUMN_4_5_NULL = true;
- public final static Boolean COLUMN_4_5_UNIQUE = false;
public final static Boolean COLUMN_4_5_AUTO_GENERATED = false;
public final static String COLUMN_4_5_FOREIGN_KEY = null;
public final static String COLUMN_4_5_CHECK = null;
@@ -2139,7 +2150,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_6_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_6_DATE_FORMAT = null;
public final static Boolean COLUMN_4_6_NULL = true;
- public final static Boolean COLUMN_4_6_UNIQUE = false;
public final static Boolean COLUMN_4_6_AUTO_GENERATED = false;
public final static String COLUMN_4_6_FOREIGN_KEY = null;
public final static String COLUMN_4_6_CHECK = null;
@@ -2159,7 +2169,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_7_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_7_DATE_FORMAT = null;
public final static Boolean COLUMN_4_7_NULL = true;
- public final static Boolean COLUMN_4_7_UNIQUE = false;
public final static Boolean COLUMN_4_7_AUTO_GENERATED = false;
public final static String COLUMN_4_7_FOREIGN_KEY = null;
public final static String COLUMN_4_7_CHECK = null;
@@ -2179,7 +2188,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_8_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_8_DATE_FORMAT = null;
public final static Boolean COLUMN_4_8_NULL = true;
- public final static Boolean COLUMN_4_8_UNIQUE = false;
public final static Boolean COLUMN_4_8_AUTO_GENERATED = false;
public final static String COLUMN_4_8_FOREIGN_KEY = null;
public final static String COLUMN_4_8_CHECK = null;
@@ -2199,7 +2207,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_9_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_9_DATE_FORMAT = null;
public final static Boolean COLUMN_4_9_NULL = true;
- public final static Boolean COLUMN_4_9_UNIQUE = false;
public final static Boolean COLUMN_4_9_AUTO_GENERATED = false;
public final static String COLUMN_4_9_FOREIGN_KEY = null;
public final static String COLUMN_4_9_CHECK = null;
@@ -2219,7 +2226,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_10_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_10_DATE_FORMAT = null;
public final static Boolean COLUMN_4_10_NULL = true;
- public final static Boolean COLUMN_4_10_UNIQUE = false;
public final static Boolean COLUMN_4_10_AUTO_GENERATED = false;
public final static String COLUMN_4_10_FOREIGN_KEY = null;
public final static String COLUMN_4_10_CHECK = null;
@@ -2239,7 +2245,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_11_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_11_DATE_FORMAT = null;
public final static Boolean COLUMN_4_11_NULL = true;
- public final static Boolean COLUMN_4_11_UNIQUE = false;
public final static Boolean COLUMN_4_11_AUTO_GENERATED = false;
public final static String COLUMN_4_11_FOREIGN_KEY = null;
public final static String COLUMN_4_11_CHECK = null;
@@ -2259,7 +2264,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_12_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_12_DATE_FORMAT = null;
public final static Boolean COLUMN_4_12_NULL = true;
- public final static Boolean COLUMN_4_12_UNIQUE = false;
public final static Boolean COLUMN_4_12_AUTO_GENERATED = false;
public final static String COLUMN_4_12_FOREIGN_KEY = null;
public final static String COLUMN_4_12_CHECK = null;
@@ -2279,7 +2283,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_13_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_13_DATE_FORMAT = null;
public final static Boolean COLUMN_4_13_NULL = true;
- public final static Boolean COLUMN_4_13_UNIQUE = false;
public final static Boolean COLUMN_4_13_AUTO_GENERATED = false;
public final static String COLUMN_4_13_FOREIGN_KEY = null;
public final static String COLUMN_4_13_CHECK = null;
@@ -2299,7 +2302,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_14_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_14_DATE_FORMAT = null;
public final static Boolean COLUMN_4_14_NULL = true;
- public final static Boolean COLUMN_4_14_UNIQUE = false;
public final static Boolean COLUMN_4_14_AUTO_GENERATED = false;
public final static String COLUMN_4_14_FOREIGN_KEY = null;
public final static String COLUMN_4_14_CHECK = null;
@@ -2319,7 +2321,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_15_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_15_DATE_FORMAT = null;
public final static Boolean COLUMN_4_15_NULL = true;
- public final static Boolean COLUMN_4_15_UNIQUE = false;
public final static Boolean COLUMN_4_15_AUTO_GENERATED = false;
public final static String COLUMN_4_15_FOREIGN_KEY = null;
public final static String COLUMN_4_15_CHECK = null;
@@ -2339,7 +2340,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_16_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_16_DATE_FORMAT = null;
public final static Boolean COLUMN_4_16_NULL = true;
- public final static Boolean COLUMN_4_16_UNIQUE = false;
public final static Boolean COLUMN_4_16_AUTO_GENERATED = false;
public final static String COLUMN_4_16_FOREIGN_KEY = null;
public final static String COLUMN_4_16_CHECK = null;
@@ -2359,7 +2359,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_17_TYPE_DTO = ColumnTypeDto.INT;
public final static Long COLUMN_4_17_DATE_FORMAT = null;
public final static Boolean COLUMN_4_17_NULL = true;
- public final static Boolean COLUMN_4_17_UNIQUE = false;
public final static Boolean COLUMN_4_17_AUTO_GENERATED = false;
public final static String COLUMN_4_17_FOREIGN_KEY = null;
public final static String COLUMN_4_17_CHECK = null;
@@ -2379,7 +2378,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_18_TYPE_DTO = ColumnTypeDto.DECIMAL;
public final static Long COLUMN_4_18_DATE_FORMAT = null;
public final static Boolean COLUMN_4_18_NULL = true;
- public final static Boolean COLUMN_4_18_UNIQUE = false;
public final static Boolean COLUMN_4_18_AUTO_GENERATED = false;
public final static String COLUMN_4_18_FOREIGN_KEY = null;
public final static String COLUMN_4_18_CHECK = null;
@@ -2399,7 +2397,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_19_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_19_DATE_FORMAT = null;
public final static Boolean COLUMN_4_19_NULL = true;
- public final static Boolean COLUMN_4_19_UNIQUE = false;
public final static Boolean COLUMN_4_19_AUTO_GENERATED = false;
public final static String COLUMN_4_19_FOREIGN_KEY = null;
public final static String COLUMN_4_19_CHECK = null;
@@ -2419,7 +2416,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_20_TYPE_DTO = ColumnTypeDto.BOOL;
public final static Long COLUMN_4_20_DATE_FORMAT = null;
public final static Boolean COLUMN_4_20_NULL = true;
- public final static Boolean COLUMN_4_20_UNIQUE = false;
public final static Boolean COLUMN_4_20_AUTO_GENERATED = false;
public final static String COLUMN_4_20_FOREIGN_KEY = null;
public final static String COLUMN_4_20_CHECK = null;
@@ -2439,7 +2435,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_4_21_TYPE_DTO = ColumnTypeDto.DECIMAL;
public final static Long COLUMN_4_21_DATE_FORMAT = null;
public final static Boolean COLUMN_4_21_NULL = true;
- public final static Boolean COLUMN_4_21_UNIQUE = false;
public final static Boolean COLUMN_4_21_AUTO_GENERATED = false;
public final static String COLUMN_4_21_FOREIGN_KEY = null;
public final static String COLUMN_4_21_CHECK = null;
@@ -2459,7 +2454,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_5_1_TYPE_DTO = ColumnTypeDto.BIGINT;
public final static Long COLUMN_5_1_DATE_FORMAT = null;
public final static Boolean COLUMN_5_1_NULL = false;
- public final static Boolean COLUMN_5_1_UNIQUE = true;
public final static Boolean COLUMN_5_1_AUTO_GENERATED = true;
public final static String COLUMN_5_1_FOREIGN_KEY = null;
public final static String COLUMN_5_1_CHECK = null;
@@ -2478,7 +2472,6 @@ public abstract class BaseTest {
public final static Long COLUMN_5_2_SIZE = 20L;
public final static Long COLUMN_5_2_DATE_FORMAT = null;
public final static Boolean COLUMN_5_2_NULL = false;
- public final static Boolean COLUMN_5_2_UNIQUE = false;
public final static Boolean COLUMN_5_2_AUTO_GENERATED = false;
public final static String COLUMN_5_2_FOREIGN_KEY = null;
public final static String COLUMN_5_2_CHECK = null;
@@ -2497,7 +2490,6 @@ public abstract class BaseTest {
public final static Long COLUMN_5_3_SIZE = 40L;
public final static Long COLUMN_5_3_DATE_FORMAT = null;
public final static Boolean COLUMN_5_3_NULL = false;
- public final static Boolean COLUMN_5_3_UNIQUE = false;
public final static Boolean COLUMN_5_3_AUTO_GENERATED = false;
public final static String COLUMN_5_3_FOREIGN_KEY = null;
public final static String COLUMN_5_3_CHECK = null;
@@ -2515,7 +2507,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_5_4_TYPE_DTO = ColumnTypeDto.YEAR;
public final static Long COLUMN_5_4_DATE_FORMAT = null;
public final static Boolean COLUMN_5_4_NULL = true;
- public final static Boolean COLUMN_5_4_UNIQUE = false;
public final static Boolean COLUMN_5_4_AUTO_GENERATED = false;
public final static String COLUMN_5_4_FOREIGN_KEY = null;
public final static String COLUMN_5_4_CHECK = null;
@@ -2533,7 +2524,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_5_5_TYPE_DTO = ColumnTypeDto.TIME;
public final static Long COLUMN_5_5_DATE_FORMAT = null;
public final static Boolean COLUMN_5_5_NULL = true;
- public final static Boolean COLUMN_5_5_UNIQUE = false;
public final static Boolean COLUMN_5_5_AUTO_GENERATED = false;
public final static String COLUMN_5_5_FOREIGN_KEY = null;
public final static String COLUMN_5_5_CHECK = null;
@@ -2551,7 +2541,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_5_6_TYPE_DTO = ColumnTypeDto.BIGINT;
public final static Long COLUMN_5_6_DATE_FORMAT = null;
public final static Boolean COLUMN_5_6_NULL = true;
- public final static Boolean COLUMN_5_6_UNIQUE = false;
public final static Boolean COLUMN_5_6_AUTO_GENERATED = false;
public final static String COLUMN_5_6_FOREIGN_KEY = null;
public final static String COLUMN_5_6_CHECK = null;
@@ -2569,7 +2558,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_8_1_TYPE_DTO = ColumnTypeDto.BIGINT;
public final static Long COLUMN_8_1_DATE_FORMAT = null;
public final static Boolean COLUMN_8_1_NULL = false;
- public final static Boolean COLUMN_8_1_UNIQUE = true;
public final static Boolean COLUMN_8_1_AUTO_GENERATED = true;
public final static String COLUMN_8_1_FOREIGN_KEY = null;
public final static String COLUMN_8_1_CHECK = null;
@@ -2587,7 +2575,6 @@ public abstract class BaseTest {
public final static ColumnTypeDto COLUMN_8_2_TYPE_DTO = ColumnTypeDto.INT;
public final static Long COLUMN_8_2_DATE_FORMAT = null;
public final static Boolean COLUMN_8_2_NULL = true;
- public final static Boolean COLUMN_8_2_UNIQUE = false;
public final static Boolean COLUMN_8_2_AUTO_GENERATED = false;
public final static String COLUMN_8_2_FOREIGN_KEY = null;
public final static String COLUMN_8_2_CHECK = null;
@@ -3105,9 +3092,11 @@ public abstract class BaseTest {
.build());
public final static Long TABLE_1_FOREIGN_KEY_1_ID = 1L;
+ public final static String TABLE_1_FOREIGN_KEY_1_NAME = "FK_JUNIT_1";
public final static ForeignKey TABLE_1_FOREIGN_KEY_1 = ForeignKey.builder()
.fkid(TABLE_1_FOREIGN_KEY_1_ID)
+ .name(TABLE_1_FOREIGN_KEY_1_NAME)
.referencedTable(TABLE_2)
.table(TABLE_1)
.references(List.of()) /* TABLE_1_FOREIGN_KEY_REFERENCE */
@@ -3125,6 +3114,7 @@ public abstract class BaseTest {
public final static Constraints TABLE_1_CONSTRAINTS = Constraints.builder()
.foreignKeys(List.of(TABLE_1_FOREIGN_KEY_1))
.uniques(List.of(Unique.builder()
+ .name("UK_1")
.columns(List.of(
TABLE_1_COLUMNS.get(0),
TABLE_1_COLUMNS.get(1)
@@ -3136,6 +3126,7 @@ public abstract class BaseTest {
public final static Constraints TABLE_2_CONSTRAINTS = Constraints.builder()
.uniques(List.of(Unique.builder()
+ .name("UK_1")
.columns(List.of(TABLE_2_COLUMNS.get(0)))
.table(TABLE_2)
.build()))
@@ -4122,6 +4113,7 @@ public abstract class BaseTest {
public final static Constraints TABLE_3_CONSTRAINTS = Constraints.builder()
.uniques(List.of(Unique.builder()
+ .name("UK_1")
.columns(List.of(TABLE_3_COLUMNS.get(0)))
.table(TABLE_3)
.build()))
@@ -4638,7 +4630,10 @@ public abstract class BaseTest {
.build());
public final static Constraints TABLE_5_CONSTRAINTS = Constraints.builder()
- .uniques(List.of(Unique.builder().columns(List.of(TABLE_5_COLUMNS.get(0))).build()))
+ .uniques(List.of(Unique.builder()
+ .name("UK_1")
+ .columns(List.of(TABLE_5_COLUMNS.get(0)))
+ .build()))
.build();
public final static List<ForeignKeyCreateDto> TABLE_5_FOREIGN_KEYS_INVALID_CREATE = List.of(ForeignKeyCreateDto.builder()
@@ -4768,7 +4763,10 @@ public abstract class BaseTest {
.build());
public final static Constraints TABLE_6_CONSTRAINTS = Constraints.builder()
- .uniques(List.of(Unique.builder().columns(List.of(TABLE_6_COLUMNS.get(0))).build()))
+ .uniques(List.of(Unique.builder()
+ .name("UK_1")
+ .columns(List.of(TABLE_6_COLUMNS.get(0)))
+ .build()))
.build();
public final static List<ColumnCreateDto> TABLE_6_COLUMNS_CREATE = List.of(
@@ -4845,7 +4843,6 @@ public abstract class BaseTest {
public final static TableColumnType COLUMN_6_1_TYPE = TableColumnType.BIGINT;
public final static Long COLUMN_6_1_DATE_FORMAT = null;
public final static Boolean COLUMN_6_1_NULL = false;
- public final static Boolean COLUMN_6_1_UNIQUE = false;
public final static Boolean COLUMN_6_1_AUTO_GENERATED = false;
public final static String COLUMN_6_1_FOREIGN_KEY = null;
public final static String COLUMN_6_1_CHECK = null;
@@ -4862,7 +4859,6 @@ public abstract class BaseTest {
public final static TableColumnType COLUMN_6_2_TYPE = TableColumnType.BIGINT;
public final static Long COLUMN_6_2_DATE_FORMAT = null;
public final static Boolean COLUMN_6_2_NULL = false;
- public final static Boolean COLUMN_6_2_UNIQUE = false;
public final static Boolean COLUMN_6_2_AUTO_GENERATED = false;
public final static String COLUMN_6_2_FOREIGN_KEY = null;
public final static String COLUMN_6_2_CHECK = null;
@@ -4908,21 +4904,6 @@ public abstract class BaseTest {
public final static String VIEW_1_QUERY = "select `location`, `lat`, `lng` from `weather_location`";
public final static String VIEW_1_QUERY_HASH = "dc81a6877c7c51a6a6f406e1fc2a255e44a0d49a20548596e0d583c3eb849c23";
- public final static List<ViewColumn> VIEW_1_COLUMNS = List.of(
- ViewColumn.builder()
- .ordinalPosition(0)
- .column(TABLE_2_COLUMNS.get(0))
- .build(),
- ViewColumn.builder()
- .ordinalPosition(1)
- .column(TABLE_2_COLUMNS.get(1))
- .build(),
- ViewColumn.builder()
- .ordinalPosition(2)
- .column(TABLE_2_COLUMNS.get(2))
- .build()
- );
-
public final static List<ColumnDto> VIEW_1_COLUMNS_DTO = List.of(
TABLE_2_COLUMNS_DTO.get(0),
TABLE_2_COLUMNS_DTO.get(1),
@@ -4939,9 +4920,30 @@ public abstract class BaseTest {
.query(VIEW_1_QUERY)
.queryHash(VIEW_1_QUERY_HASH)
.createdBy(USER_1_ID)
- .columns(VIEW_1_COLUMNS)
+ .columns(null) /* VIEW_1_COLUMNS */
.build();
+ public final static List<ViewColumn> VIEW_1_COLUMNS = List.of(
+ ViewColumn.builder()
+ .id(1L)
+ .ordinalPosition(0)
+ .column(TABLE_2_COLUMNS.get(0))
+ .view(VIEW_1)
+ .build(),
+ ViewColumn.builder()
+ .id(2L)
+ .ordinalPosition(1)
+ .column(TABLE_2_COLUMNS.get(1))
+ .view(VIEW_1)
+ .build(),
+ ViewColumn.builder()
+ .id(3L)
+ .ordinalPosition(2)
+ .column(TABLE_2_COLUMNS.get(2))
+ .view(VIEW_1)
+ .build()
+ );
+
public final static ViewDto VIEW_1_DTO = ViewDto.builder()
.id(VIEW_1_ID)
.isInitialView(VIEW_1_INITIAL_VIEW)
@@ -4980,28 +4982,9 @@ public abstract class BaseTest {
public final static Long VIEW_2_CONTAINER_ID = CONTAINER_1_ID;
public final static Long VIEW_2_DATABASE_ID = DATABASE_1_ID;
public final static Boolean VIEW_2_PUBLIC = true;
- public final static String VIEW_2_QUERY = "select `date`, `location` as loc, `rainfall`, `mintemp` from `weather_aus` where `location` = 'Albury'";
+ public final static String VIEW_2_QUERY = "select `date`, `location` as loc, `location`, `rainfall`, `mintemp` from `weather_aus` where `location` = 'Albury'";
public final static String VIEW_2_QUERY_HASH = "987fc946772ffb6d85060262dcb5df419692a1f6772ea995e3dedb53c191e984";
- public final static List<ViewColumn> VIEW_2_COLUMNS = List.of(
- ViewColumn.builder()
- .ordinalPosition(0)
- .column(TABLE_1_COLUMNS.get(1))
- .build(),
- ViewColumn.builder()
- .ordinalPosition(1)
- .column(TABLE_1_COLUMNS.get(2))
- .build(),
- ViewColumn.builder()
- .ordinalPosition(2)
- .column(TABLE_1_COLUMNS.get(4))
- .build(),
- ViewColumn.builder()
- .ordinalPosition(3)
- .column(TABLE_1_COLUMNS.get(3))
- .build()
- );
-
public final static List<ColumnDto> VIEW_2_COLUMNS_DTO = List.of(
TABLE_1_COLUMNS_DTO.get(1),
TABLE_1_COLUMNS_DTO.get(2),
@@ -5016,12 +4999,40 @@ public abstract class BaseTest {
.internalName(VIEW_2_INTERNAL_NAME)
.vdbid(VIEW_2_DATABASE_ID)
.isPublic(VIEW_2_PUBLIC)
- .columns(VIEW_2_COLUMNS)
+ .columns(null) /* VIEW_2_COLUMNS */
.query(VIEW_2_QUERY)
.queryHash(VIEW_2_QUERY_HASH)
.createdBy(USER_1_ID)
.build();
+ public final static List<ViewColumn> VIEW_2_COLUMNS = List.of(
+ ViewColumn.builder()
+ .id(4L)
+ .ordinalPosition(0)
+ .column(TABLE_1_COLUMNS.get(1))
+ .view(VIEW_2)
+ .build(),
+ ViewColumn.builder()
+ .id(5L)
+ .ordinalPosition(1)
+ .alias("loc")
+ .column(TABLE_1_COLUMNS.get(2))
+ .view(VIEW_2)
+ .build(),
+ ViewColumn.builder()
+ .id(6L)
+ .ordinalPosition(2)
+ .column(TABLE_1_COLUMNS.get(4))
+ .view(VIEW_2)
+ .build(),
+ ViewColumn.builder()
+ .id(7L)
+ .ordinalPosition(3)
+ .column(TABLE_1_COLUMNS.get(3))
+ .view(VIEW_2)
+ .build()
+ );
+
public final static ViewDto VIEW_2_DTO = ViewDto.builder()
.id(VIEW_2_ID)
.isInitialView(VIEW_2_INITIAL_VIEW)
@@ -5057,25 +5068,6 @@ public abstract class BaseTest {
public final static String VIEW_3_QUERY = "select w.`mintemp`, w.`rainfall`, w.`location`, m.`date` from `weather_aus` w join `junit2` m on m.`location` = w.`location` and m.`date` = w.`date`";
public final static String VIEW_3_QUERY_HASH = "bbbaa56a5206b3dc3e6cf9301b0db9344eb6f19b100c7b88550ffb597a0bd255";
- public final static List<ViewColumn> VIEW_3_COLUMNS = List.of(
- ViewColumn.builder()
- .ordinalPosition(0)
- .column(TABLE_1_COLUMNS.get(3))
- .build(),
- ViewColumn.builder()
- .ordinalPosition(1)
- .column(TABLE_1_COLUMNS.get(4))
- .build(),
- ViewColumn.builder()
- .ordinalPosition(2)
- .column(TABLE_1_COLUMNS.get(2))
- .build(),
- ViewColumn.builder()
- .ordinalPosition(3)
- .column(TABLE_1_COLUMNS.get(1))
- .build()
- );
-
public final static List<ColumnDto> VIEW_3_COLUMNS_DTO = List.of(
TABLE_1_COLUMNS_DTO.get(3),
TABLE_1_COLUMNS_DTO.get(4),
@@ -5090,12 +5082,39 @@ public abstract class BaseTest {
.internalName(VIEW_3_INTERNAL_NAME)
.vdbid(VIEW_3_DATABASE_ID)
.isPublic(VIEW_3_PUBLIC)
- .columns(VIEW_3_COLUMNS)
+ .columns(null) /* VIEW_3_COLUMNS */
.query(VIEW_3_QUERY)
.queryHash(VIEW_3_QUERY_HASH)
.createdBy(USER_1_ID)
.build();
+ public final static List<ViewColumn> VIEW_3_COLUMNS = List.of(
+ ViewColumn.builder()
+ .id(8L)
+ .ordinalPosition(0)
+ .column(TABLE_1_COLUMNS.get(3))
+ .view(VIEW_3)
+ .build(),
+ ViewColumn.builder()
+ .id(9L)
+ .ordinalPosition(1)
+ .column(TABLE_1_COLUMNS.get(4))
+ .view(VIEW_3)
+ .build(),
+ ViewColumn.builder()
+ .id(10L)
+ .ordinalPosition(2)
+ .column(TABLE_1_COLUMNS.get(2))
+ .view(VIEW_3)
+ .build(),
+ ViewColumn.builder()
+ .id(11L)
+ .ordinalPosition(3)
+ .column(TABLE_1_COLUMNS.get(1))
+ .view(VIEW_3)
+ .build()
+ );
+
public final static ViewDto VIEW_3_DTO = ViewDto.builder()
.id(VIEW_3_ID)
.isInitialView(VIEW_3_INITIAL_VIEW)
@@ -5133,110 +5152,144 @@ public abstract class BaseTest {
public final static String VIEW_4_QUERY = "SELECT `animal_name`, `hair`, `feathers`, `eggs`, `milk`, `airborne`, `aquatic`, `predator`, `backbone`, `breathes`, `venomous`, `fins`, `legs`, `tail`, `domestic`, `catsize`, `class_type` FROM `zoo` WHERE `class_type` = 1";
public final static String VIEW_4_QUERY_HASH = "3561cd0bb0b0e94d6f15ae602134252a5760d09d660a71a4fb015b6991c8ba0b";
+ public final static List<ColumnDto> VIEW_4_COLUMNS_DTO = List.of(
+ TABLE_5_COLUMNS_DTO.get(1),
+ TABLE_5_COLUMNS_DTO.get(2),
+ TABLE_5_COLUMNS_DTO.get(3),
+ TABLE_5_COLUMNS_DTO.get(5),
+ TABLE_5_COLUMNS_DTO.get(6),
+ TABLE_5_COLUMNS_DTO.get(8),
+ TABLE_5_COLUMNS_DTO.get(10),
+ TABLE_5_COLUMNS_DTO.get(11),
+ TABLE_5_COLUMNS_DTO.get(12),
+ TABLE_5_COLUMNS_DTO.get(13),
+ TABLE_5_COLUMNS_DTO.get(14),
+ TABLE_5_COLUMNS_DTO.get(15),
+ TABLE_5_COLUMNS_DTO.get(16),
+ TABLE_5_COLUMNS_DTO.get(17),
+ TABLE_5_COLUMNS_DTO.get(18),
+ TABLE_5_COLUMNS_DTO.get(19),
+ TABLE_5_COLUMNS_DTO.get(20)
+ );
+
+ public final static View VIEW_4 = View.builder()
+ .id(VIEW_4_ID)
+ .isInitialView(VIEW_4_INITIAL_VIEW)
+ .name(VIEW_4_NAME)
+ .internalName(VIEW_4_INTERNAL_NAME)
+ .vdbid(VIEW_4_DATABASE_ID)
+ .isPublic(VIEW_4_PUBLIC)
+ .query(VIEW_4_QUERY)
+ .queryHash(VIEW_4_QUERY_HASH)
+ .createdBy(USER_1_ID)
+ .columns(null) /* VIEW_4_COLUMNS */
+ .build();
+
public final static List<ViewColumn> VIEW_4_COLUMNS = List.of(
ViewColumn.builder()
+ .id(12L)
.ordinalPosition(0)
.column(TABLE_5_COLUMNS.get(1))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(13L)
.ordinalPosition(1)
.column(TABLE_5_COLUMNS.get(2))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(14L)
.ordinalPosition(2)
.column(TABLE_5_COLUMNS.get(3))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(15L)
.ordinalPosition(3)
.column(TABLE_5_COLUMNS.get(5))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(16L)
.ordinalPosition(4)
.column(TABLE_5_COLUMNS.get(6))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(17L)
.ordinalPosition(5)
.column(TABLE_5_COLUMNS.get(8))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(18L)
.ordinalPosition(6)
.column(TABLE_5_COLUMNS.get(10))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(19L)
.ordinalPosition(7)
.column(TABLE_5_COLUMNS.get(11))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(20L)
.ordinalPosition(8)
.column(TABLE_5_COLUMNS.get(12))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(21L)
.ordinalPosition(9)
.column(TABLE_5_COLUMNS.get(13))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(22L)
.ordinalPosition(10)
.column(TABLE_5_COLUMNS.get(14))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(23L)
.ordinalPosition(11)
.column(TABLE_5_COLUMNS.get(15))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(24L)
.ordinalPosition(12)
.column(TABLE_5_COLUMNS.get(16))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(25L)
.ordinalPosition(13)
.column(TABLE_5_COLUMNS.get(17))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(26L)
.ordinalPosition(14)
.column(TABLE_5_COLUMNS.get(18))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(27L)
.ordinalPosition(15)
.column(TABLE_5_COLUMNS.get(19))
+ .view(VIEW_4)
.build(),
ViewColumn.builder()
+ .id(28L)
.ordinalPosition(16)
.column(TABLE_5_COLUMNS.get(20))
+ .view(VIEW_4)
.build()
);
- public final static List<ColumnDto> VIEW_4_COLUMNS_DTO = List.of(
- TABLE_5_COLUMNS_DTO.get(1),
- TABLE_5_COLUMNS_DTO.get(2),
- TABLE_5_COLUMNS_DTO.get(3),
- TABLE_5_COLUMNS_DTO.get(5),
- TABLE_5_COLUMNS_DTO.get(6),
- TABLE_5_COLUMNS_DTO.get(8),
- TABLE_5_COLUMNS_DTO.get(10),
- TABLE_5_COLUMNS_DTO.get(11),
- TABLE_5_COLUMNS_DTO.get(12),
- TABLE_5_COLUMNS_DTO.get(13),
- TABLE_5_COLUMNS_DTO.get(14),
- TABLE_5_COLUMNS_DTO.get(15),
- TABLE_5_COLUMNS_DTO.get(16),
- TABLE_5_COLUMNS_DTO.get(17),
- TABLE_5_COLUMNS_DTO.get(18),
- TABLE_5_COLUMNS_DTO.get(19),
- TABLE_5_COLUMNS_DTO.get(20)
- );
-
- public final static View VIEW_4 = View.builder()
- .id(VIEW_4_ID)
- .isInitialView(VIEW_4_INITIAL_VIEW)
- .name(VIEW_4_NAME)
- .internalName(VIEW_4_INTERNAL_NAME)
- .vdbid(VIEW_4_DATABASE_ID)
- .isPublic(VIEW_4_PUBLIC)
- .query(VIEW_4_QUERY)
- .queryHash(VIEW_4_QUERY_HASH)
- .createdBy(USER_1_ID)
- .columns(VIEW_4_COLUMNS)
- .build();
-
public final static ViewDto VIEW_4_DTO = ViewDto.builder()
.id(VIEW_4_ID)
.isInitialView(VIEW_4_INITIAL_VIEW)
@@ -5270,6 +5323,7 @@ public abstract class BaseTest {
.query(VIEW_5_QUERY)
.queryHash(VIEW_5_QUERY_HASH)
.createdBy(USER_1_ID)
+ .columns(null)
.build();
public final static Long QUERY_1_RESULT_ID = 1L;
diff --git a/dbrepo-ui/README.md b/dbrepo-ui/README.md
index fbd83426ca01b7b4acb8aad17690bf577cd73945..5717916121eedacd881ee913a6e7fb9b2d4e9bb6 100644
--- a/dbrepo-ui/README.md
+++ b/dbrepo-ui/README.md
@@ -40,4 +40,15 @@ Optionally, generate a coverage report:
```bash
yarn run coverage
-```
\ No newline at end of file
+```
+
+## Troubleshooting
+
+Watchpack Error (watcher): Error: ENOSPC: System limit for number of file watchers reached,
+watch `./dbrepo-ui/node_modules/...`
+
+* Cause: Started the local development server with `yarn dev` and the system file watchers could not be created since
+ the maximum limit is reached, debug with `cat /proc/sys/fs/inotify/max_user_watches`.
+* Solution: Increase the limit with `sudo sysctl fs.inotify.max_user_watches=131070` and verify
+ success: `sudo sysctl -p`
+* See further: [https://stackoverflow.com/questions/53930305/nodemon-error-system-limit-for-number-of-file-watchers-reached](https://stackoverflow.com/questions/53930305/nodemon-error-system-limit-for-number-of-file-watchers-reached)
diff --git a/dbrepo-ui/pages/database/_database_id/table/_table_id/data.vue b/dbrepo-ui/pages/database/_database_id/table/_table_id/data.vue
index c663a38827699e4d6bb262d9e4db8979ae6e2298..d19a2525c5e1956d253f62f34b49b1ad35998679 100644
--- a/dbrepo-ui/pages/database/_database_id/table/_table_id/data.vue
+++ b/dbrepo-ui/pages/database/_database_id/table/_table_id/data.vue
@@ -249,7 +249,7 @@ export default {
this.table.columns.map((c) => {
return {
value: c.internal_name,
- text: c.name,
+ text: c.internal_name,
sortable: false
}
}).forEach(header => this.headers.push(header))
diff --git a/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue b/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue
index 980b440ddd6beaa3a89283e350f5b6a657bc88da..331ca955fa666d43abc2868d2c318b9cb9022711 100644
--- a/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue
+++ b/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue
@@ -54,6 +54,16 @@
label="Value quotes" />
</v-col>
</v-row>
+ <v-row dense>
+ <v-col cols="8">
+ <v-text-field
+ v-model="tableImport.line_termination"
+ hint="Representation of a new line"
+ placeholder="e.g. \r\n"
+ clearable
+ label="Line termination" />
+ </v-col>
+ </v-row>
<v-row dense>
<v-col cols="8">
<v-text-field
@@ -143,8 +153,9 @@ export default {
quote: '"',
false_element: null,
true_element: null,
- null_element: 'NA',
+ null_element: '',
separator: ',',
+ line_termination: '\\r\\n',
skip_lines: 1
},
file: {
diff --git a/dbrepo-ui/pages/database/_database_id/table/_table_id/schema.vue b/dbrepo-ui/pages/database/_database_id/table/_table_id/schema.vue
index ccddb4a1d8cf971268ad1d8b4c2da547ec7a110d..6b79b38ec344755ea56b6518c6594c2e4d1baeab 100644
--- a/dbrepo-ui/pages/database/_database_id/table/_table_id/schema.vue
+++ b/dbrepo-ui/pages/database/_database_id/table/_table_id/schema.vue
@@ -3,7 +3,7 @@
<TableToolbar :selection="selection" />
<v-toolbar color="secondary white--text" flat>
<strong>
- <v-toolbar-title v-text="title" />
+ <v-toolbar-title>Schema</v-toolbar-title>
</strong>
</v-toolbar>
<v-card tile>
@@ -64,6 +64,29 @@
</template>
</v-data-table>
</v-card>
+ <v-card v-if="hasConstraints" tile>
+ <v-card-subtitle>Constraints</v-card-subtitle>
+ <v-card-text>
+ <ul>
+ <li v-for="(foreignKey,i) in table.constraints.foreign_keys" :key="`fk-${i}`">
+ <strong>FOREIGN KEY</strong>
+ <span v-text="foreignKey.name" />
+ (<i v-text="foreignKeyColumns(foreignKey)" />)
+ <strong>REFERENCES</strong>
+ <a :href="`/database/${database.id}/table/${foreignKey.referenced_table.id}/schema`" v-text="foreignKeyReferencedTable(foreignKey)" />
+ (<i v-text="foreignKeyReferencedColumns(foreignKey)" />)
+ </li>
+ <li v-for="(uniqueConstraint,i) in table.constraints.uniques" :key="`uk-${i}`">
+ <strong>UNIQUE INDEX</strong>
+ (<i v-text="uniqueColumns(uniqueConstraint)" />)
+ </li>
+ <li v-for="(checkConstraint,i) in table.constraints.checks" :key="`uk-${i}`">
+ <strong>CHECK CONSTRAINT</strong>
+ (<i v-text="checkConstraint" />)
+ </li>
+ </ul>
+ </v-card-text>
+ </v-card>
<v-dialog
v-if="table && database"
v-model="dialogSemantic"
@@ -81,7 +104,6 @@
</template>
<script>
import TableToolbar from '@/components/table/TableToolbar.vue'
-import TableService from '@/api/table.service'
export default {
components: {
@@ -106,7 +128,6 @@ export default {
{ value: 'column_concept', text: 'Concept' },
{ value: 'column_unit', text: 'Unit' },
{ value: 'is_primary_key', text: 'Primary Key' },
- { value: 'unique', text: 'Unique' },
{ value: 'is_null_allowed', text: 'Nullable' },
{ value: 'auto_generated', text: 'Sequence' }
],
@@ -129,12 +150,6 @@ export default {
roles () {
return this.$store.state.roles
},
- title () {
- if (!this.table) {
- return null
- }
- return this.table.constraints.checks.length > 0 ? `Constraints: ${this.table.constraints.checks}` : 'Schema'
- },
canAssignSemanticInformation () {
if (!this.user) {
return false
@@ -147,23 +162,11 @@ export default {
}
return this.roles.includes('modify-table-column-semantics') && (this.access.type === 'write_all' || this.table.owner.username === this.user.username)
},
- versionColor () {
- if (this.version === null) {
- return 'secondary white--text'
- }
- return 'primary white--text'
- },
- versionFormatted () {
- if (this.version === null) {
- return null
- }
- return this.version + ' (UTC)'
- },
- versionISO () {
- if (this.version === null) {
- return null
+ hasConstraints () {
+ if (!this.table || !this.table.constraints) {
+ return false
}
- return this.version.substring(0, 10) + 'T' + this.version.substring(11, 19) + 'Z'
+ return this.table.constraints.uniques.length > 0 || this.table.constraints.checks.length > 0 || this.table.constraints.foreign_keys.length > 0
}
},
mounted () {
@@ -177,13 +180,6 @@ export default {
const uniqueColumnIds = this.table.constraints.uniques.map(u => u.columns.map(c => c.id)).flat()
return uniqueColumnIds.includes(column.id)
},
- columnName (column) {
- const filter = this.columnTypes.filter(t => t.value === column.column_type)
- if (filter.length > 0) {
- return filter[0].text
- }
- return column.column_type
- },
extra (column) {
if (['date', 'datetime', 'timestamp', 'time'].includes(column.column_type)) {
return `fsp=${column.date_format.unix_format}`
@@ -219,21 +215,29 @@ export default {
}
this.dialogSemantic = false
},
- loadTable () {
- if (!this.$route.params.database_id || !this.$route.params.table_id) {
- return
+ foreignKeyColumns (foreignKey) {
+ if (!foreignKey) {
+ return null
+ }
+ return foreignKey.columns.map(c => c.internal_name).join(',')
+ },
+ foreignKeyReferencedTable (foreignKey) {
+ if (!foreignKey) {
+ return null
+ }
+ return foreignKey.referenced_table.internal_name
+ },
+ foreignKeyReferencedColumns (foreignKey) {
+ if (!foreignKey) {
+ return null
+ }
+ return foreignKey.referenced_columns.map(c => c.internal_name).join(',')
+ },
+ uniqueColumns (uniqueConstraint) {
+ if (!uniqueConstraint) {
+ return null
}
- this.loading = true
- TableService.findOne(this.$route.params.database_id, this.$route.params.table_id)
- .then((table) => {
- this.$store.commit('SET_TABLE', table)
- })
- .catch(() => {
- this.loading = false
- })
- .finally(() => {
- this.loading = false
- })
+ return uniqueConstraint.columns.map(c => c.internal_name).join(',')
}
}
}
diff --git a/dbrepo-ui/pages/database/_database_id/table/import.vue b/dbrepo-ui/pages/database/_database_id/table/import.vue
index a013fcff66234e4bf2a5348714140e35a5263944..4faa62a52079bf686674cf54f3123bfcb0a25a60 100644
--- a/dbrepo-ui/pages/database/_database_id/table/import.vue
+++ b/dbrepo-ui/pages/database/_database_id/table/import.vue
@@ -88,6 +88,16 @@
label="Value quotes" />
</v-col>
</v-row>
+ <v-row dense>
+ <v-col cols="8">
+ <v-text-field
+ v-model="tableImport.line_termination"
+ hint="Representation of a new line"
+ placeholder="e.g. \r\n"
+ clearable
+ label="Line termination" />
+ </v-col>
+ </v-row>
<v-row dense>
<v-col cols="8">
<v-text-field
@@ -262,8 +272,9 @@ export default {
quote: '"',
false_element: null,
true_element: null,
- null_element: 'NA',
+ null_element: '',
separator: ',',
+ line_termination: '\\r\\n',
skip_lines: 1
},
loading: false,