diff --git a/dbrepo-metadata-db/setup-schema.sql b/dbrepo-metadata-db/setup-schema.sql index bfc22b830134621bf189858054240858d9c18dec..fe452045ca24d999b88f4554bf66b97b3b7101a3 100644 --- a/dbrepo-metadata-db/setup-schema.sql +++ b/dbrepo-metadata-db/setup-schema.sql @@ -160,6 +160,11 @@ CREATE TABLE IF NOT EXISTS `mdb_columns` d INT, auto_generated BOOLEAN DEFAULT false, is_null_allowed BOOLEAN NOT NULL DEFAULT true, + val_min NUMERIC NULL, + val_max NUMERIC NULL, + mean NUMERIC NULL, + median NUMERIC NULL, + std_dev Numeric NULL, created timestamp NOT NULL DEFAULT NOW(), last_modified timestamp, FOREIGN KEY (tID) REFERENCES mdb_tables (ID), @@ -195,23 +200,6 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_nom` PRIMARY KEY (tID, cID) ) WITH SYSTEM VERSIONING; -CREATE TABLE IF NOT EXISTS `mdb_columns_num` -( - tID bigint, - cID bigint, - SIunit TEXT, - MaxVal NUMERIC, - MinVal NUMERIC, - Mean NUMERIC, - Median NUMERIC, - Sd Numeric, - -- Histogram INTEGER[], - last_modified timestamp, - created timestamp NOT NULL DEFAULT NOW(), - FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID), - PRIMARY KEY (tID, cID) -) WITH SYSTEM VERSIONING; - CREATE TABLE IF NOT EXISTS `mdb_columns_cat` ( tID bigint, @@ -546,7 +534,8 @@ VALUES (1, '%Y-%c-%d %H:%i:%S.%f', 'yyyy-MM-dd HH:mm:ss.SSSSSS', '2022-01-30 13: INSERT INTO `mdb_ontologies` (prefix, uri, uri_pattern, sparql_endpoint, rdf_path) VALUES ('om', 'http://www.ontology-of-units-of-measure.org/resource/om-2/', 'http://www.ontology-of-units-of-measure.org/resource/om-2/.*', null, 'rdf/om-2.0.rdf'), - ('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql', null), + ('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql', + null), ('mo', 'http://purl.org/ontology/mo/', 'http://purl.org/ontology/mo/.*', null, null), ('dc', 'http://purl.org/dc/elements/1.1/', null, null, null), ('xsd', 'http://www.w3.org/2001/XMLSchema#', null, null, null), diff --git a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java index eb748f1ec48dbceeff66923a056bfb125eee2c3a..bf5e9fe292249b8c19610828d46a0d32c8cebc72 100644 --- a/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java +++ b/dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java @@ -6,6 +6,7 @@ import at.tuwien.api.database.table.columns.concepts.UnitDto; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.persistence.Column; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; import lombok.*; @@ -15,6 +16,7 @@ import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; +import java.math.BigDecimal; import java.util.List; @Getter @@ -91,6 +93,26 @@ public class ColumnDto { @Field(name = "d", type = FieldType.Integer) private Integer d; + @Schema(example = "0") + @Field(name = "val_min", type = FieldType.Double) + private BigDecimal valMin; + + @Schema(example = "100") + @Field(name = "val_max", type = FieldType.Double) + private BigDecimal valMax; + + @Schema(example = "45.4") + @Field(name = "mean", type = FieldType.Double) + private BigDecimal mean; + + @Schema(example = "51") + @Field(name = "median", type = FieldType.Double) + private BigDecimal median; + + @Schema(example = "5.32") + @Field(name = "std_dev", type = FieldType.Double) + private BigDecimal stdDev; + @Field(name = "concept", includeInParent = true, type = FieldType.Nested) private ConceptDto concept; diff --git a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java index 8c4dd227c1dbfd5643eb07cdb08468bcbe56fbb5..05fd24b66e7f858b47327caf226d2903733ed535 100644 --- a/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java +++ b/dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java @@ -12,6 +12,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener; import jakarta.persistence.*; +import java.math.BigDecimal; import java.time.Instant; import java.util.List; @@ -116,6 +117,21 @@ public class TableColumn implements Comparable<TableColumn> { @Column private Integer d; + @Column(name = "val_min") + private BigDecimal valMin; + + @Column(name = "val_max") + private BigDecimal valMax; + + @Column + private BigDecimal mean; + + @Column + private BigDecimal median; + + @Column(name = "std_dev") + private BigDecimal stdDev; + @LastModifiedDate @Column(columnDefinition = "TIMESTAMP") private Instant lastModified; diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java index 8330501cb2db3edc236527e3f2117bba05537fb0..4cf3bf7459721e47a0d4c852a19bf8f772f5b96e 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java @@ -20,12 +20,14 @@ import at.tuwien.service.SemanticService; import at.tuwien.service.TableService; import at.tuwien.service.UserService; import at.tuwien.utils.UserUtil; +import ch.qos.logback.core.testUtil.RandomUtil; import com.mchange.v2.c3p0.ComboPooledDataSource; import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import java.math.BigDecimal; import java.security.Principal; import java.sql.Connection; import java.sql.PreparedStatement; @@ -33,6 +35,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; import java.util.Optional; +import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; @Log4j2 @Service @@ -244,6 +248,25 @@ public class TableServiceImpl extends HibernateConnector implements TableService final TableColumn column = findColumn(table, columnId); /* assign */ if (updateDto.getUnitUri() != null) { + // FIXME BEGIN democode + if (updateDto.getConceptUri().equals("http://www.wikidata.org/entity/Q11466")) { + if (updateDto.getUnitUri().equals("http://www.ontology-of-units-of-measure.org/resource/om-2/degreeFahrenheit")) { + column.setValMin(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(32, 68))) /* 0-20 */; + column.setValMax(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(67, 104)).add(column.getValMin())) /* 20-40 */; + column.setMean(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(32, 104))); + column.setMedian(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(32, 104))); + column.setStdDev(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 20))); + log.warn("Faked demo statistical values (deg. Fahrenheit) for column with id {}", columnId); + } else if (updateDto.getUnitUri().equals("http://www.ontology-of-units-of-measure.org/resource/om-2/degreeCelsius")) { + column.setValMin(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 20))) /* 0-20 */; + column.setValMax(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(20, 40)).add(column.getValMin())) /* 20-40 */; + column.setMean(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 40))); + column.setMedian(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 40))); + column.setStdDev(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 10))); + log.warn("Faked demo statistical values (deg. Celsius) for column with id {}", columnId); + } + } + // FIXME END democode try { column.setUnit(semanticService.findUnit(updateDto.getUnitUri())); log.debug("Found unit with uri {} in metadata database", updateDto.getUnitUri()); diff --git a/dbrepo-search-db/init/indices/column.json b/dbrepo-search-db/init/indices/column.json index f0654c92e7a4a4de579552d8f3d849905ce9f8e4..ca5f82d2115030b0a01d077e6ef49df891489ace 100644 --- a/dbrepo-search-db/init/indices/column.json +++ b/dbrepo-search-db/init/indices/column.json @@ -33,6 +33,21 @@ "d": { "type": "integer" }, + "val_min": { + "type": "double" + }, + "val_max": { + "type": "double" + }, + "mean": { + "type": "double" + }, + "median": { + "type": "double" + }, + "std_dev": { + "type": "double" + }, "database_id": { "type": "keyword" }, diff --git a/dbrepo-search-service/app/api/routes.py b/dbrepo-search-service/app/api/routes.py index b6d45b3091867b94f40f6118cf7a048b909a695d..21afad4e76eff81662b282504c11114279712283 100644 --- a/dbrepo-search-service/app/api/routes.py +++ b/dbrepo-search-service/app/api/routes.py @@ -153,7 +153,11 @@ def search(): logging.debug('search request body: %s', req_body) search_term = req_body.get("search_term") t1 = req_body.get("t1") + if not str(t1).isdigit(): + t1 = None t2 = req_body.get("t2") + if not str(t2).isdigit(): + t2 = None fieldValuePairs = req_body.get("field_value_pairs") response = general_search(search_term, t1, t2, fieldValuePairs) return response, 200 diff --git a/dbrepo-search-service/app/opensearch_client.py b/dbrepo-search-service/app/opensearch_client.py index 0b454b05d0b26a8d44c61253f607aa3a68ecbf7b..c98ab001a85a62ee3be86f7cbbf4221084f5e126 100644 --- a/dbrepo-search-service/app/opensearch_client.py +++ b/dbrepo-search-service/app/opensearch_client.py @@ -173,31 +173,21 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None): ) response["status"] = 200 return response - if t1 is not None: - logging.debug(f"query has start value {t1} present") - time_range_query = { - "range": { - "created": { - "gte": t1, - "lte": t2, - } - } - } - queries.append(time_range_query) - if t1 is not None and t2 is not None: - logging.debug(f"query has start value {t1} and end value {t2} present") - time_range_query = { - "range": { - "created": { - "gte": t1, - "lte": t2, - } - } - } - queries.append(time_range_query) if fieldValuePairs is not None and len(fieldValuePairs) > 0: - logging.debug('query has fieldValuePairs present') + logging.debug('query has field_value_pairs present') musts = [] + is_range_open_end = False + is_range_open_begin = False + is_range_query = False + if t1 is not None and t2 is None: + is_range_open_begin = True + logging.debug(f"query has only start value {t1} present") + if t1 is None and t2 is not None: + is_range_open_end = True + logging.debug(f"query has only end value {t2} present") + if t1 is not None and t2 is not None: + is_range_query = True + logging.debug(f"query has start value {t1} and end value {t2} present") for key, value in fieldValuePairs.items(): if key == "type" and value in searchable_indices: logging.debug("search for specific index: %s", value) @@ -209,11 +199,49 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None): logging.debug( f"field name {key} starts with index name {index}: flattened field name to {new_field}") key = new_field - musts.append({ - "match": { - key: {"query": value, "minimum_should_match": "90%"} - } - }) + if is_range_open_end and re.match(f"unit\\.", key): + logging.debug(f"omit key={key} because query type=open end range and key is somewhat unit") + logging.info(f"add match-query for range ),{t2}]") + musts.append({ + "range": { + "val_max": { + "lte": t2 + } + } + }) + elif is_range_open_begin and re.match(f"unit\\.", key): + logging.debug(f"omit key={key} because query type=open begin range and key is somewhat unit") + logging.info(f"add match-query for range [{t1},(") + musts.append({ + "range": { + "val_min": { + "gte": t1 + } + } + }) + elif is_range_query and re.match(f"unit\\.", key): + logging.debug(f"omit key={key} because query type=full range and key is somewhat unit") + logging.info(f"add match-query for range [{t1},{t2}]") + musts.append({ + "range": { + "val_min": { + "gte": t1 + } + } + }) + musts.append({ + "range": { + "val_max": { + "lte": t2 + } + } + }) + else: + musts.append({ + "match": { + key: {"query": value, "minimum_should_match": "90%"} + } + }) specific_query = {"bool": {"must": musts}} queries.append(specific_query) body = { diff --git a/dbrepo-ui/components/search/AdvancedSearch.vue b/dbrepo-ui/components/search/AdvancedSearch.vue index 507d66f9f3f6dd4bfdb599f24e291dc56fa5e469..df30313ee2893d1a6f71b3cba950ac2666e32e27 100644 --- a/dbrepo-ui/components/search/AdvancedSearch.vue +++ b/dbrepo-ui/components/search/AdvancedSearch.vue @@ -180,6 +180,7 @@ export default { return } this.resetAdvancedSearchFields() + this.$emit('search-result', []) this.loadingFields = true SearchService.getFields(newType) .then((response) => {