From 01bd7ac348298962199b22db2b29cdef5460921f Mon Sep 17 00:00:00 2001 From: Martin Weise <martin.weise@tuwien.ac.at> Date: Mon, 24 Feb 2025 16:40:55 +0100 Subject: [PATCH] Finished migration Signed-off-by: Martin Weise <martin.weise@tuwien.ac.at> --- .gitlab-ci.yml | 2 +- dbrepo-metadata-db/migration/16/data.py | 106 ++++++++++-------- .../migration/16/requirements.txt | 2 +- dbrepo-metadata-db/migration/16/schema.sql | 5 +- .../api/container/image/DataTypeDto.java | 3 + .../api/container/image/OperatorDto.java | 1 + .../api/database/table/columns/ColumnDto.java | 6 +- .../api/database/table/columns/EnumDto.java | 27 +++++ .../api/database/table/columns/SetDto.java | 27 +++++ .../at/tuwien/entities/database/Database.java | 1 + .../database/table/columns/ColumnEnum.java | 35 ++++++ .../database/table/columns/ColumnSet.java | 35 ++++++ .../database/table/columns/TableColumn.java | 16 +-- .../java/at/tuwien/mapper/MetadataMapper.java | 3 + .../tuwien/repository/DatabaseRepository.java | 2 + .../at/tuwien/endpoints/DatabaseEndpoint.java | 23 ++-- .../at/tuwien/endpoints/TableEndpoint.java | 2 +- .../at/tuwien/service/DatabaseService.java | 2 + .../service/impl/DatabaseServiceImpl.java | 5 + .../tuwien/service/impl/TableServiceImpl.java | 20 +++- .../main/java/at/tuwien/test/BaseTest.java | 16 --- lib/python/dbrepo/RestClient.py | 61 +++++++++- lib/python/dbrepo/api/dto.py | 35 +++++- lib/python/pyproject.toml | 2 +- lib/python/setup.py | 2 +- lib/python/tests/test_unit_ontology.py | 35 ++++++ 26 files changed, 377 insertions(+), 97 deletions(-) create mode 100644 dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/EnumDto.java create mode 100644 dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/SetDto.java create mode 100644 dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/ColumnEnum.java create mode 100644 dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/ColumnSet.java create mode 100644 lib/python/tests/test_unit_ontology.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0cfcd600cd..6dafaa52e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -403,7 +403,7 @@ test-lib: script: - "pip install pipenv" - "pipenv install gunicorn && pipenv install --dev --system --deploy" - - cd ./lib/python/ && coverage run -m pytest tests/test_unit_analyse.py tests/test_unit_container.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_image.py tests/test_unit_messages.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt + - cd ./lib/python/ && coverage run -m pytest tests/test_unit_analyse.py tests/test_unit_ontology.py tests/test_unit_container.py tests/test_unit_database.py tests/test_unit_identifier.py tests/test_unit_image.py tests/test_unit_messages.py tests/test_unit_license.py tests/test_unit_query.py tests/test_unit_rest_client.py tests/test_unit_table.py tests/test_unit_user.py tests/test_unit_view.py --junitxml=report.xml && coverage html --omit="test/*" && coverage report --omit="test/*" > ./coverage.txt - "cat ./coverage.txt | grep -o 'TOTAL[^%]*%'" artifacts: when: always diff --git a/dbrepo-metadata-db/migration/16/data.py b/dbrepo-metadata-db/migration/16/data.py index f15aad1036..fe627c2bef 100644 --- a/dbrepo-metadata-db/migration/16/data.py +++ b/dbrepo-metadata-db/migration/16/data.py @@ -26,7 +26,10 @@ def update_concepts() -> None: def update_ontologies() -> None: plan.append("-- ontologies") plan.append("BEGIN;") - plan.append(f"UPDATE mdb_ontologies SET id = UUID();") + for ontology in client.get_ontologies(): + old_id = ontology.id + new_id: uuid = uuid.uuid4() + plan.append(f"UPDATE mdb_ontologies SET id = '{new_id}' WHERE id = '{old_id}';") plan.append("COMMIT;") @@ -44,13 +47,22 @@ def update_units() -> None: def update_images() -> None: plan.append("-- images") plan.append("BEGIN;") - for image in client.get_images(): - old_id: int = image.id + for _image in client.get_images(): + old_id: int = _image.id + image = client.get_image(old_id) new_id: uuid = uuid.uuid4() plan.append(f"UPDATE mdb_images SET id = '{new_id}' WHERE id = '{old_id}';") - plan.append(f"UPDATE mdb_image_operators SET id = UUID(), image_id = '{new_id}' WHERE image_id = '{old_id}';") - plan.append(f"UPDATE mdb_image_types SET id = UUID(), image_id = '{new_id}' WHERE image_id = '{old_id}';") - plan.append(f"UPDATE mdb_containers SET id = UUID(), image_id = '{new_id}' WHERE image_id = '{old_id}';") + plan.append(f"UPDATE mdb_image_operators SET image_id = '{new_id}' WHERE image_id = '{old_id}';") + plan.append(f"UPDATE mdb_image_types SET image_id = '{new_id}' WHERE image_id = '{old_id}';") + plan.append(f"UPDATE mdb_containers SET image_id = '{new_id}' WHERE image_id = '{old_id}';") + for operator in image.operators: + o_old_id: int = operator.id + o_new_id: uuid = uuid.uuid4() + plan.append(f"UPDATE mdb_image_operators SET id = '{o_new_id}' WHERE id = '{o_old_id}';") + for data_type in image.data_types: + d_old_id: int = data_type.id + d_new_id: uuid = uuid.uuid4() + plan.append(f"UPDATE mdb_image_types SET id = '{d_new_id}' WHERE id = '{d_old_id}';") plan.append("COMMIT;") @@ -86,6 +98,9 @@ def update_databases() -> None: tbl_old_id: int = table.id tbl_new_id: uuid = uuid.uuid4() plan.append(f"UPDATE mdb_identifiers SET tid = '{tbl_new_id}' WHERE tid = '{tbl_old_id}';") + plan.append(f"UPDATE mdb_columns SET tID = '{tbl_new_id}' WHERE tID = '{tbl_old_id}';") + plan.append(f"UPDATE mdb_constraints_primary_key SET pkid = UUID(), tID = '{tbl_new_id}' WHERE tID = '{tbl_old_id}';") + plan.append(f"UPDATE mdb_constraints_unique SET tid = '{tbl_new_id}' WHERE tid = '{tbl_old_id}';") plan.append( f"UPDATE mdb_constraints_checks SET id = UUID(), tid = '{tbl_new_id}' WHERE tid = '{tbl_old_id}';") for fk in table.constraints.foreign_keys: @@ -122,6 +137,14 @@ def update_databases() -> None: plan.append(f"UPDATE mdb_columns_units SET cID = '{col_new_id}' WHERE cID = '{col_old_id}';") plan.append(f"UPDATE mdb_columns_sets SET column_id = '{col_new_id}' WHERE column_id = '{col_old_id}';") plan.append(f"UPDATE mdb_columns_enums SET column_id = '{col_new_id}' WHERE column_id = '{col_old_id}';") + for set in column.sets: + s_old_id: int = set.id + s_new_id: uuid = uuid.uuid4() + plan.append(f"UPDATE mdb_columns_sets SET id = '{s_new_id}' WHERE id = '{s_old_id}';") + for enum in column.enums: + e_old_id: int = enum.id + e_new_id: uuid = uuid.uuid4() + plan.append(f"UPDATE mdb_columns_enums SET id = '{e_new_id}' WHERE id = '{e_old_id}';") plan.append(f"UPDATE mdb_tables SET ID = '{tbl_new_id}' WHERE ID = '{tbl_old_id}';") plan.append(f"UPDATE mdb_databases SET id = '{new_id}' WHERE id = '{old_id}';") plan.append("COMMIT;") @@ -130,60 +153,50 @@ def update_databases() -> None: def update_messages() -> None: plan.append("-- messages") plan.append("BEGIN;") - plan.append(f"UPDATE mdb_messages SET ID = UUID();") + for message in client.get_messages(): + old_id = message.id + new_id: uuid = uuid.uuid4() + plan.append(f"UPDATE mdb_messages SET id = '{new_id}' WHERE id = '{old_id}';") plan.append("COMMIT;") def update_identifiers() -> None: plan.append("-- identifiers") plan.append("BEGIN;") - for identified in client.get_identifiers(): - i_old_id: int = identified.id + for _identifier in client.get_identifiers(): + identifier = client.get_identifier(identifier_id=_identifier.id) + i_old_id: int = identifier.id i_new_id: uuid = uuid.uuid4() plan.append(f"UPDATE mdb_identifiers SET ID = '{i_new_id}' WHERE id = '{i_old_id}';") - plan.append(f"UPDATE mdb_identifier_creators SET id = UUID(), pid = '{i_new_id}' WHERE pid = '{i_old_id}';") - plan.append(f"UPDATE mdb_identifier_descriptions SET id = UUID(), pid = '{i_new_id}' WHERE pid = '{i_old_id}';") - plan.append(f"UPDATE mdb_identifier_titles SET id = UUID(), pid = '{i_new_id}' WHERE pid = '{i_old_id}';") - plan.append(f"UPDATE mdb_identifier_funders SET id = UUID(), pid = '{i_new_id}' WHERE pid = '{i_old_id}';") + plan.append(f"UPDATE mdb_identifier_titles SET pid = '{i_new_id}' WHERE pid = '{i_old_id}';") + plan.append(f"UPDATE mdb_identifier_descriptions SET pid = '{i_new_id}' WHERE pid = '{i_old_id}';") + plan.append(f"UPDATE mdb_identifier_creators SET pid = '{i_new_id}' WHERE pid = '{i_old_id}';") + plan.append(f"UPDATE mdb_identifier_funders SET pid = '{i_new_id}' WHERE pid = '{i_old_id}';") plan.append(f"UPDATE mdb_identifier_licenses SET pid = '{i_new_id}' WHERE pid = '{i_old_id}';") + for title in identifier.titles: + t_old_id = title.id + t_new_id: uuid = uuid.uuid4() + plan.append(f"UPDATE mdb_identifier_titles SET id = '{t_new_id}' WHERE id = '{t_old_id}';") + for description in identifier.descriptions: + d_old_id = description.id + d_new_id: uuid = uuid.uuid4() + plan.append(f"UPDATE mdb_identifier_descriptions SET id = '{d_new_id}' WHERE id = '{d_old_id}';") + for creator in identifier.creators: + c_old_id = creator.id + c_new_id: uuid = uuid.uuid4() + plan.append(f"UPDATE mdb_identifier_creators SET id = '{c_new_id}' WHERE id = '{c_old_id}';") + for funder in identifier.funders: + f_old_id = funder.id + f_new_id: uuid = uuid.uuid4() + plan.append(f"UPDATE mdb_identifier_funders SET id = '{f_new_id}' WHERE id = '{f_old_id}';") plan.append("COMMIT;") -def finish_schema() -> None: - plan.append(f"ALTER TABLE `mdb_ontologies` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_units` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_concepts` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_messages` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_image_operators` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_image_types` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_access` ADD PRIMARY KEY (aUserID, aDBID);") - plan.append(f"ALTER TABLE `mdb_have_access` ADD PRIMARY KEY (user_id, database_id);") - plan.append(f"ALTER TABLE `mdb_identifier_creators` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_identifier_descriptions` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_identifier_funders` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_identifier_licenses` ADD PRIMARY KEY (pid, license_id);") - plan.append(f"ALTER TABLE `mdb_identifier_titles` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_related_identifiers` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_identifiers` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_columns_concepts` ADD PRIMARY KEY (id, cid);") - plan.append(f"ALTER TABLE `mdb_columns_enums` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_columns_sets` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_columns_units` ADD PRIMARY KEY (id, cID);") - plan.append(f"ALTER TABLE `mdb_constraints_checks` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_constraints_foreign_key_reference` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_constraints_foreign_key` ADD PRIMARY KEY (fkid);") - plan.append(f"ALTER TABLE `mdb_constraints_primary_key` ADD PRIMARY KEY (pkid);") - plan.append(f"ALTER TABLE `mdb_constraints_unique` ADD PRIMARY KEY (uid);") - plan.append(f"ALTER TABLE `mdb_constraints_unique_columns` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_columns` ADD PRIMARY KEY (ID);") - plan.append(f"ALTER TABLE `mdb_tables` ADD PRIMARY KEY (ID);") - plan.append(f"ALTER TABLE `mdb_view_columns` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_view` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_databases` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_containers` ADD PRIMARY KEY (id);") - plan.append(f"ALTER TABLE `mdb_images` ADD PRIMARY KEY (id);") if __name__ == '__main__': plan.append("SET FOREIGN_KEY_CHECKS=0;") + plan.append("BEGIN;") + plan.append(f"INSERT INTO mdb_have_access SELECT uu.id as user_id, d.id as database_id, 'WRITE_ALL' as access_type, NOW() as created FROM mdb_databases d, mdb_users uu WHERE NOT EXISTS(SELECT 1 FROM mdb_have_access a JOIN mdb_users u ON a.user_id = u.id AND u.is_internal = TRUE) AND uu.is_internal = TRUE;") + plan.append("COMMIT;") update_concepts() update_units() update_messages() @@ -192,5 +205,6 @@ if __name__ == '__main__': update_containers() update_databases() update_identifiers() + update_messages() plan.append("SET FOREIGN_KEY_CHECKS=1;") print("\n".join(plan)) diff --git a/dbrepo-metadata-db/migration/16/requirements.txt b/dbrepo-metadata-db/migration/16/requirements.txt index b3ba05dd27..f7e737b2ce 100644 --- a/dbrepo-metadata-db/migration/16/requirements.txt +++ b/dbrepo-metadata-db/migration/16/requirements.txt @@ -1 +1 @@ -dbrepo==1.6.5rc10 \ No newline at end of file +dbrepo==1.6.5rc15 \ No newline at end of file diff --git a/dbrepo-metadata-db/migration/16/schema.sql b/dbrepo-metadata-db/migration/16/schema.sql index b0f623d78a..68a87f1a67 100644 --- a/dbrepo-metadata-db/migration/16/schema.sql +++ b/dbrepo-metadata-db/migration/16/schema.sql @@ -33,8 +33,6 @@ ALTER TABLE mdb_messages CHANGE COLUMN id id VARCHAR(36) NOT NULL DEFAULT UUID(); ALTER TABLE `mdb_messages` DROP PRIMARY KEY; -ALTER TABLE `mdb_messages` - ADD SYSTEM VERSIONING; -- mdb_image_operators ALTER TABLE mdb_image_operators DROP SYSTEM VERSIONING; @@ -613,3 +611,6 @@ ALTER TABLE mdb_ontologies ADD PRIMARY KEY (id); ALTER TABLE mdb_ontologies ADD SYSTEM VERSIONING; +-- mdb_messages +ALTER TABLE `mdb_messages` + ADD SYSTEM VERSIONING; \ No newline at end of file diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/DataTypeDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/DataTypeDto.java index a4215c0392..693a8eb2fe 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/DataTypeDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/DataTypeDto.java @@ -17,6 +17,9 @@ import lombok.extern.jackson.Jacksonized; @ToString public class DataTypeDto { + @NotNull + private Long id; + @NotBlank @JsonProperty("display_name") @Schema(example = "TIME(fsp)") diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/OperatorDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/OperatorDto.java index 96113b5347..820472d332 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/OperatorDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/container/image/OperatorDto.java @@ -17,6 +17,7 @@ import lombok.extern.jackson.Jacksonized; @ToString public class OperatorDto { + @NotNull private Long id; @NotBlank diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java index 462821b53c..40f19f544a 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java @@ -119,12 +119,10 @@ public class ColumnDto { @Schema(example = "false") private Boolean isNullAllowed; - @Schema(example = "[\"val1\"]") @Parameter(description = "enum values, only considered when type = ENUM") - private List<String> enums; + private List<EnumDto> enums; - @Schema(example = "[\"val1\"]") @Parameter(description = "enum values, only considered when type = ENUM") - private List<String> sets; + private List<SetDto> sets; } diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/EnumDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/EnumDto.java new file mode 100644 index 0000000000..166e1a0037 --- /dev/null +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/EnumDto.java @@ -0,0 +1,27 @@ +package at.tuwien.api.database.table.columns; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; +import lombok.*; +import lombok.extern.jackson.Jacksonized; + +@Getter +@Setter +@Builder +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +@Jacksonized +@ToString +public class EnumDto { + + @NotNull + @Schema(example = "1") + private Long id; + + @NotNull + @Schema(example = "3") + private String value; + +} diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/SetDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/SetDto.java new file mode 100644 index 0000000000..d6849f21f0 --- /dev/null +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/SetDto.java @@ -0,0 +1,27 @@ +package at.tuwien.api.database.table.columns; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; +import lombok.*; +import lombok.extern.jackson.Jacksonized; + +@Getter +@Setter +@Builder +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +@Jacksonized +@ToString +public class SetDto { + + @NotNull + @Schema(example = "1") + private Long id; + + @NotNull + @Schema(example = "3") + private String value; + +} diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/Database.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/Database.java index d90a702db3..b7619e3b24 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/Database.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/Database.java @@ -33,6 +33,7 @@ import static jakarta.persistence.GenerationType.IDENTITY; }) @NamedQueries({ @NamedQuery(name = "Database.findAllDesc", query = "select distinct d from Database d order by d.id desc"), + @NamedQuery(name = "Database.findAllByInternalNameDesc", query = "select distinct d from Database d where d.internalName = ?1 order by d.id desc"), @NamedQuery(name = "Database.findAllAtLestReadAccessDesc", query = "select distinct d from Database d where exists(select a.hdbid from DatabaseAccess a where a.huserid = ?1 and a.hdbid = d.id) order by d.id desc"), @NamedQuery(name = "Database.findAllPublicOrSchemaPublicDesc", query = "select distinct d from Database d where d.isPublic = true or d.isSchemaPublic = true order by d.id desc"), @NamedQuery(name = "Database.findAllPublicOrSchemaPublicOrReadAccessDesc", query = "select distinct d from Database d where d.isPublic = true or d.isSchemaPublic = true or exists(select a.hdbid from DatabaseAccess a where a.huserid = ?1 and a.hdbid = d.id) order by d.id desc"), diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/ColumnEnum.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/ColumnEnum.java new file mode 100644 index 0000000000..323ada8109 --- /dev/null +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/ColumnEnum.java @@ -0,0 +1,35 @@ +package at.tuwien.entities.database.table.columns; + +import jakarta.persistence.*; +import lombok.*; + +import static jakarta.persistence.GenerationType.IDENTITY; + +@Data +@Entity +@Builder(toBuilder = true) +@ToString +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode +@jakarta.persistence.Table(name = "mdb_columns_enums", uniqueConstraints = { + @UniqueConstraint(columnNames = {"value"}) +}) +public class ColumnEnum { + + @Id + @GeneratedValue(strategy = IDENTITY) + @Column(updatable = false, nullable = false) + private Long id; + + @ToString.Exclude + @EqualsAndHashCode.Exclude + @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE}) + @JoinColumns({ + @JoinColumn(name = "column_id", referencedColumnName = "id", nullable = false) + }) + private TableColumn column; + + @Column(columnDefinition = "VARCHAR(255)") + private String value; +} diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/ColumnSet.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/ColumnSet.java new file mode 100644 index 0000000000..9ffd697c7a --- /dev/null +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/ColumnSet.java @@ -0,0 +1,35 @@ +package at.tuwien.entities.database.table.columns; + +import jakarta.persistence.*; +import lombok.*; + +import static jakarta.persistence.GenerationType.IDENTITY; + +@Data +@Entity +@Builder(toBuilder = true) +@ToString +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode +@Table(name = "mdb_columns_sets", uniqueConstraints = { + @UniqueConstraint(columnNames = {"value"}) +}) +public class ColumnSet { + + @Id + @GeneratedValue(strategy = IDENTITY) + @Column(updatable = false, nullable = false) + private Long id; + + @ToString.Exclude + @EqualsAndHashCode.Exclude + @ManyToOne(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE}) + @JoinColumns({ + @JoinColumn(name = "column_id", referencedColumnName = "id", nullable = false) + }) + private TableColumn column; + + @Column(columnDefinition = "VARCHAR(255)") + private String value; +} diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java index 9f4c4e0606..85eac6eafc 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java @@ -4,6 +4,8 @@ import at.tuwien.entities.database.table.Table; import com.fasterxml.jackson.annotation.JsonFormat; import jakarta.persistence.*; import lombok.*; +import org.hibernate.annotations.OnDelete; +import org.hibernate.annotations.OnDeleteAction; import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; import org.springframework.data.jpa.domain.support.AuditingEntityListener; @@ -83,15 +85,13 @@ public class TableColumn implements Comparable<TableColumn> { inverseJoinColumns = @JoinColumn(name = "id", referencedColumnName = "id")) private TableColumnUnit unit; - @ElementCollection(fetch = FetchType.LAZY, targetClass = String.class) - @CollectionTable(name = "mdb_columns_enums", joinColumns = @JoinColumn(name = "column_id")) - @Column(name = "value", nullable = false) - private List<String> enums; + @OnDelete(action = OnDeleteAction.CASCADE) + @OneToMany(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE, CascadeType.PERSIST}, mappedBy = "column") + private List<ColumnEnum> enums; - @ElementCollection(fetch = FetchType.LAZY, targetClass = String.class) - @CollectionTable(name = "mdb_columns_sets", joinColumns = @JoinColumn(name = "column_id")) - @Column(name = "value", nullable = false) - private List<String> sets; + @OnDelete(action = OnDeleteAction.CASCADE) + @OneToMany(fetch = FetchType.LAZY, cascade = {CascadeType.MERGE, CascadeType.PERSIST}, mappedBy = "column") + private List<ColumnSet> sets; @Column private Long size; diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java index cff05b8d6f..adced5a577 100644 --- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java +++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/mapper/MetadataMapper.java @@ -57,6 +57,7 @@ import at.tuwien.entities.container.image.ContainerImage; import at.tuwien.entities.container.image.DataType; import at.tuwien.entities.database.*; import at.tuwien.entities.database.table.Table; +import at.tuwien.entities.database.table.columns.ColumnEnum; import at.tuwien.entities.database.table.columns.TableColumn; import at.tuwien.entities.database.table.columns.TableColumnConcept; import at.tuwien.entities.database.table.columns.TableColumnUnit; @@ -766,6 +767,8 @@ public interface MetadataMapper { @Mapping(target = "isNullAllowed", source = "data.nullAllowed"), @Mapping(target = "name", source = "data.name"), @Mapping(target = "internalName", expression = "java(nameToInternalName(data.getName()))"), + @Mapping(target = "enums", ignore = true), + @Mapping(target = "sets", ignore = true), }) TableColumn columnCreateDtoToTableColumn(CreateTableColumnDto data, ContainerImage image); diff --git a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/repository/DatabaseRepository.java b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/repository/DatabaseRepository.java index 38a0a80441..97b1555d47 100644 --- a/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/repository/DatabaseRepository.java +++ b/dbrepo-metadata-service/repositories/src/main/java/at/tuwien/repository/DatabaseRepository.java @@ -23,4 +23,6 @@ public interface DatabaseRepository extends JpaRepository<Database, Long> { List<Database> findAllPublicOrSchemaPublicByInternalNameDesc(String internalName); + List<Database> findAllByInternalNameDesc(String internalName); + } diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/DatabaseEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/DatabaseEndpoint.java index 41cc59ca6a..294d3299f2 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/DatabaseEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/DatabaseEndpoint.java @@ -79,15 +79,24 @@ public class DatabaseEndpoint extends AbstractEndpoint { final List<Database> databases; if (principal != null) { if (internalName != null) { - log.debug("filter request to contain only public databases or where user with id {} has at least read access that match internal name {}", getId(principal), internalName); - databases = databaseService.findAllPublicOrSchemaPublicOrReadAccessByInternalName(getId(principal), internalName); + if (isSystem(principal)) { + log.debug("filter request to contain only databases that match internal name: {}", internalName); + databases = databaseService.findByInternalName(internalName); + } else { + log.debug("filter request to contain only public databases or where user with id {} has at least read access that match internal name: {}", getId(principal), internalName); + databases = databaseService.findAllPublicOrSchemaPublicOrReadAccessByInternalName(getId(principal), internalName); + } } else { - log.debug("filter request to contain only databases where user with id {} has at least read access", getId(principal)); - databases = databaseService.findAllPublicOrSchemaPublicOrReadAccess(getId(principal)); + if (isSystem(principal)) { + databases = databaseService.findAll(); + } else { + log.debug("filter request to contain only databases where user with id {} has at least read access", getId(principal)); + databases = databaseService.findAllPublicOrSchemaPublicOrReadAccess(getId(principal)); + } } } else { if (internalName != null) { - log.debug("filter request to contain only public databases that match internal name {}", internalName); + log.debug("filter request to contain only public databases that match internal name: {}", internalName); databases = databaseService.findAllPublicOrSchemaPublicByInternalName(internalName); } else { log.debug("filter request to contain only public databases"); @@ -521,14 +530,14 @@ public class DatabaseEndpoint extends AbstractEndpoint { .size(); database.setTables(database.getTables() .stream() - .filter(t -> t.getIsPublic() || t.getIsSchemaPublic() || optional.isPresent()) + .filter(t -> t.getIsPublic() || t.getIsSchemaPublic() || optional.isPresent() || isSystem(principal)) .toList()); log.trace("filtered database tables from {} to {}", tables, database.getTables().size()); final int views = database.getViews() .size(); database.setViews(database.getViews() .stream() - .filter(v -> v.getIsPublic() || v.getIsSchemaPublic() || optional.isPresent()) + .filter(v -> v.getIsPublic() || v.getIsSchemaPublic() || optional.isPresent() || isSystem(principal)) .toList()); log.trace("filtered database views from {} to {}", views, database.getViews().size()); if (!isSystem(principal) && !database.getOwner().getId().equals(getId(principal))) { diff --git a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java index 08535fde69..65fd63e035 100644 --- a/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java +++ b/dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java @@ -98,7 +98,7 @@ public class TableEndpoint extends AbstractEndpoint { endpointValidator.validateOnlyPrivateSchemaHasRole(database, principal, "list-tables"); return ResponseEntity.ok(database.getTables() .stream() - .filter(t -> t.getIsPublic() || t.getIsSchemaPublic()) + .filter(t -> t.getIsPublic() || t.getIsSchemaPublic() || (principal != null && isSystem(principal))) .map(metadataMapper::tableToTableBriefDto) .collect(Collectors.toList())); } diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java index 4e3765fd6e..8c0c5c806d 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/DatabaseService.java @@ -23,6 +23,8 @@ public interface DatabaseService { List<Database> findAllPublicOrSchemaPublic(); + List<Database> findByInternalName(String internalName); + List<Database> findAllPublicOrSchemaPublicOrReadAccessByInternalName(UUID userId, String internalName); /** diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceImpl.java index 8b4c73fb2f..7c310757cd 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceImpl.java @@ -62,6 +62,11 @@ public class DatabaseServiceImpl implements DatabaseService { return databaseRepository.findAllPublicOrSchemaPublicDesc(); } + @Override + public List<Database> findByInternalName(String internalName) { + return databaseRepository.findAllByInternalNameDesc(internalName); + } + @Override public List<Database> findAllPublicOrSchemaPublicOrReadAccessByInternalName(UUID userId, String internalName) { return databaseRepository.findAllPublicOrSchemaPublicOrReadAccessByInternalNameDesc(userId, internalName); diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java index da92fb7ef5..73897208d9 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java @@ -3,15 +3,13 @@ package at.tuwien.service.impl; import at.tuwien.api.database.table.CreateTableDto; import at.tuwien.api.database.table.TableStatisticDto; import at.tuwien.api.database.table.TableUpdateDto; -import at.tuwien.api.database.table.columns.CreateTableColumnDto; import at.tuwien.api.database.table.columns.ColumnStatisticDto; +import at.tuwien.api.database.table.columns.CreateTableColumnDto; import at.tuwien.api.database.table.columns.concepts.ColumnSemanticsUpdateDto; import at.tuwien.config.RabbitConfig; import at.tuwien.entities.database.Database; import at.tuwien.entities.database.table.Table; -import at.tuwien.entities.database.table.columns.TableColumn; -import at.tuwien.entities.database.table.columns.TableColumnConcept; -import at.tuwien.entities.database.table.columns.TableColumnUnit; +import at.tuwien.entities.database.table.columns.*; import at.tuwien.entities.user.User; import at.tuwien.exception.*; import at.tuwien.gateway.DataServiceGateway; @@ -119,6 +117,20 @@ public class TableServiceImpl implements TableService { for (int i = 0; i < data.getColumns().size(); i++) { final CreateTableColumnDto c = data.getColumns().get(i); final TableColumn column = metadataMapper.columnCreateDtoToTableColumn(c, database.getContainer().getImage()); + column.setEnums(c.getEnums() + .stream() + .map(e -> ColumnEnum.builder() + .column(column) + .value(e) + .build()) + .toList()); + column.setSets(c.getSets() + .stream() + .map(e -> ColumnSet.builder() + .column(column) + .value(e) + .build()) + .toList()); column.setOrdinalPosition(idx[0]++); column.setTable(table); if (c.getUnitUri() != null) { diff --git a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java index 7440de878b..8e22d2325a 100644 --- a/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java +++ b/dbrepo-metadata-service/test/src/main/java/at/tuwien/test/BaseTest.java @@ -6,8 +6,6 @@ import at.tuwien.api.amqp.ExchangeDto; import at.tuwien.api.amqp.GrantVirtualHostPermissionsDto; import at.tuwien.api.amqp.QueueDto; import at.tuwien.api.auth.CreateUserDto; -import at.tuwien.api.auth.LoginRequestDto; -import at.tuwien.api.auth.RefreshTokenRequestDto; import at.tuwien.api.container.ContainerBriefDto; import at.tuwien.api.container.ContainerDto; import at.tuwien.api.container.image.*; @@ -279,10 +277,6 @@ public abstract class BaseTest { .scope(TOKEN_ACCESS_SCOPE) .build(); - public static final RefreshTokenRequestDto REFRESH_TOKEN_REQUEST_DTO = RefreshTokenRequestDto.builder() - .refreshToken("ey.yee.skrr") - .build(); - public static final Long CONCEPT_1_ID = 1L; public static final String CONCEPT_1_NAME = "precipitation"; public static final String CONCEPT_1_URI = "http://www.wikidata.org/entity/Q25257"; @@ -446,11 +440,6 @@ public abstract class BaseTest { @SuppressWarnings("java:S2068") public static final String USER_LOCAL_ADMIN_MARIADB_PASSWORD = "*440BA4FD1A87A0999647DB67C0EE258198B247BA"; - public static final LoginRequestDto USER_LOCAL_ADMIN_LOGIN_REQUEST_DTO = LoginRequestDto.builder() - .username(USER_LOCAL_ADMIN_USERNAME) - .password(USER_LOCAL_ADMIN_PASSWORD) - .build(); - public static final UserDetails USER_LOCAL_ADMIN_DETAILS = UserDetailsDto.builder() .id(USER_LOCAL_ADMIN_ID.toString()) .username(USER_LOCAL_ADMIN_USERNAME) @@ -598,11 +587,6 @@ public abstract class BaseTest { public static final Principal USER_1_PRINCIPAL = new UsernamePasswordAuthenticationToken(USER_1_DETAILS, USER_1_PASSWORD, USER_1_DETAILS.getAuthorities()); - public static final LoginRequestDto USER_1_LOGIN_REQUEST_DTO = LoginRequestDto.builder() - .username(USER_1_USERNAME) - .password(USER_1_PASSWORD) - .build(); - public static final UUID USER_2_ID = UUID.fromString("eeb9a51b-4cd8-4039-90bf-e24f17372f7c"); public static final UUID USER_2_KEYCLOAK_ID = UUID.fromString("eeb9a51b-4cd8-4039-90bf-e24f17372f7c"); public static final String USER_2_USERNAME = "junit2"; diff --git a/lib/python/dbrepo/RestClient.py b/lib/python/dbrepo/RestClient.py index c61278a645..5db776ae54 100644 --- a/lib/python/dbrepo/RestClient.py +++ b/lib/python/dbrepo/RestClient.py @@ -1,8 +1,9 @@ import logging import os -import requests import sys import time + +import requests from pandas import DataFrame from pydantic import TypeAdapter @@ -1935,6 +1936,20 @@ class RestClient: raise ResponseCodeError(f'Failed to get licenses: response code: {response.status_code} is not ' f'200 (OK): {response.text}') + def get_ontologies(self) -> List[OntologyBrief]: + """ + Get list of ontologies. + + :returns: List of ontologies, if successful. + """ + url = f'/api/ontology' + response = self._wrapper(method="get", url=url) + if response.status_code == 200: + body = response.json() + return TypeAdapter(List[OntologyBrief]).validate_python(body) + raise ResponseCodeError(f'Failed to get ontologies: response code: {response.status_code} is not ' + f'200 (OK): {response.text}') + def get_concepts(self) -> List[ConceptBrief]: """ Get list of concepts known to the metadata database. @@ -1950,7 +1965,7 @@ class RestClient: f'200 (OK): {response.text}') def get_identifiers(self, database_id: int = None, subset_id: int = None, view_id: int = None, - table_id: int = None) -> List[IdentifierBrief] | str: + table_id: int = None) -> List[IdentifierBrief]: """ Get list of identifiers, filter by the remaining optional arguments. @@ -1992,6 +2007,48 @@ class RestClient: raise ResponseCodeError(f'Failed to get identifiers: response code: {response.status_code} is not ' f'200 (OK): {response.text}') + def get_identifier(self, identifier_id: int) -> Identifier: + """ + Get list of identifiers, filter by the remaining optional arguments. + + :param identifier_id: The identifier id. + + :returns: The identifier, if successful. + + :raises NotExistsError: If the identifier does not exist. + :raises ResponseCodeError: If something went wrong with the retrieval of the identifier. + """ + url = f'/api/identifier/{identifier_id}' + response = self._wrapper(method="get", url=url, headers={'Accept': 'application/json'}) + if response.status_code == 200: + body = response.json() + return Identifier.model_validate(body) + if response.status_code == 404: + raise NotExistsError(f'Failed to get identifier: not found') + raise ResponseCodeError(f'Failed to get identifier: response code: {response.status_code} is not ' + f'200 (OK): {response.text}') + + def get_image(self, image_id: int) -> Image: + """ + Get container image. + + :param image_id: The image id. + + :returns: The image, if successful. + + :raises NotExistsError: If the image does not exist. + :raises ResponseCodeError: If something went wrong with the retrieval of the image. + """ + url = f'/api/image/{image_id}' + response = self._wrapper(method="get", url=url, headers={'Accept': 'application/json'}) + if response.status_code == 200: + body = response.json() + return Image.model_validate(body) + if response.status_code == 404: + raise NotExistsError(f'Failed to get image: not found') + raise ResponseCodeError(f'Failed to get image: response code: {response.status_code} is not ' + f'200 (OK): {response.text}') + def get_images(self) -> List[ImageBrief] | str: """ Get list of container images. diff --git a/lib/python/dbrepo/api/dto.py b/lib/python/dbrepo/api/dto.py index fe46e6ae33..ec109606c7 100644 --- a/lib/python/dbrepo/api/dto.py +++ b/lib/python/dbrepo/api/dto.py @@ -3,9 +3,10 @@ from __future__ import annotations import datetime from dataclasses import field from enum import Enum -from pydantic import BaseModel, PlainSerializer from typing import List, Optional, Annotated +from pydantic import BaseModel, PlainSerializer + Timestamp = Annotated[ datetime.datetime, PlainSerializer(lambda v: v.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z', return_type=str) ] @@ -23,6 +24,13 @@ class JwtAuth(BaseModel): token_type: str +class Operator(BaseModel): + id: int + display_name: str + value: str + documentation: str + + class Image(BaseModel): id: int registry: str @@ -32,6 +40,7 @@ class Image(BaseModel): driver_class: str jdbc_method: str default_port: int + operators: List[Operator] = field(default_factory=list) data_types: List[DataType] = field(default_factory=list) @@ -472,6 +481,15 @@ class License(BaseModel): description: str +class OntologyBrief(BaseModel): + id: int + uri: str + prefix: str + sparql: bool + rdf: bool + uri_pattern: Optional[str] = None + + class Tuple(BaseModel): data: dict @@ -918,7 +936,18 @@ class UpdateQuery(BaseModel): persist: bool +class ColumnEnum(BaseModel): + id: int + value: str + + +class ColumnSet(BaseModel): + id: int + value: str + + class DataType(BaseModel): + id: int display_name: str value: str documentation: str @@ -953,8 +982,8 @@ class Column(BaseModel): median: Optional[float] = None concept: Optional[ConceptBrief] = None unit: Optional[UnitBrief] = None - enums: Optional[List[str]] = field(default_factory=list) - sets: Optional[List[str]] = field(default_factory=list) + enums: Optional[List[ColumnEnum]] = field(default_factory=list) + sets: Optional[List[ColumnSet]] = field(default_factory=list) index_length: Optional[int] = None length: Optional[int] = None data_length: Optional[int] = None diff --git a/lib/python/pyproject.toml b/lib/python/pyproject.toml index 60864d8e72..41ae446ed7 100644 --- a/lib/python/pyproject.toml +++ b/lib/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dbrepo" -version = "1.6.5rc10" +version = "1.6.5rc15" description = "DBRepo Python Library" keywords = [ "DBRepo", diff --git a/lib/python/setup.py b/lib/python/setup.py index acdd06508e..3de1385e8c 100644 --- a/lib/python/setup.py +++ b/lib/python/setup.py @@ -2,7 +2,7 @@ from distutils.core import setup setup(name="dbrepo", - version="1.6.5rc10", + version="1.6.5rc15", description="A library for communicating with DBRepo", url="https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/1.6/", author="Martin Weise", diff --git a/lib/python/tests/test_unit_ontology.py b/lib/python/tests/test_unit_ontology.py new file mode 100644 index 0000000000..7e37646c56 --- /dev/null +++ b/lib/python/tests/test_unit_ontology.py @@ -0,0 +1,35 @@ +import unittest + +import requests_mock + +from dbrepo.RestClient import RestClient +from dbrepo.api.dto import OntologyBrief + + +class OntologyUnitTest(unittest.TestCase): + + def test_get_ontologies_empty_succeeds(self): + with requests_mock.Mocker() as mock: + # mock + mock.get('/api/ontology', json=[]) + # test + response = RestClient().get_ontologies() + self.assertEqual([], response) + + def test_get_ontologies_succeeds(self): + with requests_mock.Mocker() as mock: + exp = [OntologyBrief(id=1, + uri="http://www.ontology-of-units-of-measure.org/resource/om-2/", + prefix="om", + sparql=False, + rdf=True, + uri_pattern="http://www.ontology-of-units-of-measure.org/resource/om-2/.*")] + # mock + mock.get('/api/ontology', json=[exp[0].model_dump()]) + # test + response = RestClient().get_ontologies() + self.assertEqual(exp, response) + + +if __name__ == "__main__": + unittest.main() -- GitLab