Added range query search

dbrepo-metadata-db/setup-schema.sql

@@ -160,6 +160,11 @@ CREATE TABLE IF NOT EXISTS `mdb_columns`
     d                INT,
     auto_generated   BOOLEAN               DEFAULT false,
     is_null_allowed  BOOLEAN      NOT NULL DEFAULT true,
+    val_min          NUMERIC      NULL,
+    val_max          NUMERIC      NULL,
+    mean             NUMERIC      NULL,
+    median           NUMERIC      NULL,
+    std_dev          Numeric      NULL,
     created          timestamp    NOT NULL DEFAULT NOW(),
     last_modified    timestamp,
     FOREIGN KEY (tID) REFERENCES mdb_tables (ID),
@@ -195,23 +200,6 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_nom`
     PRIMARY KEY (tID, cID)
 ) WITH SYSTEM VERSIONING;
 
-CREATE TABLE IF NOT EXISTS `mdb_columns_num`
-(
-    tID           bigint,
-    cID           bigint,
-    SIunit        TEXT,
-    MaxVal        NUMERIC,
-    MinVal        NUMERIC,
-    Mean          NUMERIC,
-    Median        NUMERIC,
-    Sd            Numeric,
-    --    Histogram     INTEGER[],
-    last_modified timestamp,
-    created       timestamp NOT NULL DEFAULT NOW(),
-    FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID),
-    PRIMARY KEY (tID, cID)
-) WITH SYSTEM VERSIONING;
-
 CREATE TABLE IF NOT EXISTS `mdb_columns_cat`
 (
     tID           bigint,
@@ -546,7 +534,8 @@ VALUES (1, '%Y-%c-%d %H:%i:%S.%f', 'yyyy-MM-dd HH:mm:ss.SSSSSS', '2022-01-30 13:
 INSERT INTO `mdb_ontologies` (prefix, uri, uri_pattern, sparql_endpoint, rdf_path)
 VALUES ('om', 'http://www.ontology-of-units-of-measure.org/resource/om-2/',
         'http://www.ontology-of-units-of-measure.org/resource/om-2/.*', null, 'rdf/om-2.0.rdf'),
-       ('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql', null),
+       ('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql',
+        null),
        ('mo', 'http://purl.org/ontology/mo/', 'http://purl.org/ontology/mo/.*', null, null),
        ('dc', 'http://purl.org/dc/elements/1.1/', null, null, null),
        ('xsd', 'http://www.w3.org/2001/XMLSchema#', null, null, null),

dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java

@@ -6,6 +6,7 @@ import at.tuwien.api.database.table.columns.concepts.UnitDto;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Schema;
+import jakarta.persistence.Column;
 import jakarta.validation.constraints.NotBlank;
 import jakarta.validation.constraints.NotNull;
 import lombok.*;
@@ -15,6 +16,7 @@ import org.springframework.data.elasticsearch.annotations.Document;
 import org.springframework.data.elasticsearch.annotations.Field;
 import org.springframework.data.elasticsearch.annotations.FieldType;
 
+import java.math.BigDecimal;
 import java.util.List;
 
 @Getter
@@ -91,6 +93,26 @@ public class ColumnDto {
     @Field(name = "d", type = FieldType.Integer)
     private Integer d;
 
+    @Schema(example = "0")
+    @Field(name = "val_min", type = FieldType.Double)
+    private BigDecimal valMin;
+
+    @Schema(example = "100")
+    @Field(name = "val_max", type = FieldType.Double)
+    private BigDecimal valMax;
+
+    @Schema(example = "45.4")
+    @Field(name = "mean", type = FieldType.Double)
+    private BigDecimal mean;
+
+    @Schema(example = "51")
+    @Field(name = "median", type = FieldType.Double)
+    private BigDecimal median;
+
+    @Schema(example = "5.32")
+    @Field(name = "std_dev", type = FieldType.Double)
+    private BigDecimal stdDev;
+
     @Field(name = "concept", includeInParent = true, type = FieldType.Nested)
     private ConceptDto concept;
 

dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java

@@ -12,6 +12,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
 
 import jakarta.persistence.*;
 
+import java.math.BigDecimal;
 import java.time.Instant;
 import java.util.List;
 
@@ -116,6 +117,21 @@ public class TableColumn implements Comparable<TableColumn> {
     @Column
     private Integer d;
 
+    @Column(name = "val_min")
+    private BigDecimal valMin;
+
+    @Column(name = "val_max")
+    private BigDecimal valMax;
+
+    @Column
+    private BigDecimal mean;
+
+    @Column
+    private BigDecimal median;
+
+    @Column(name = "std_dev")
+    private BigDecimal stdDev;
+
     @LastModifiedDate
     @Column(columnDefinition = "TIMESTAMP")
     private Instant lastModified;

dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java

@@ -20,12 +20,14 @@ import at.tuwien.service.SemanticService;
 import at.tuwien.service.TableService;
 import at.tuwien.service.UserService;
 import at.tuwien.utils.UserUtil;
+import ch.qos.logback.core.testUtil.RandomUtil;
 import com.mchange.v2.c3p0.ComboPooledDataSource;
 import lombok.extern.log4j.Log4j2;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
+import java.math.BigDecimal;
 import java.security.Principal;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
@@ -33,6 +35,8 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.List;
 import java.util.Optional;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
 
 @Log4j2
 @Service
@@ -244,6 +248,25 @@ public class TableServiceImpl extends HibernateConnector implements TableService
         final TableColumn column = findColumn(table, columnId);
         /* assign */
         if (updateDto.getUnitUri() != null) {
+            // FIXME BEGIN democode
+            if (updateDto.getConceptUri().equals("http://www.wikidata.org/entity/Q11466")) {
+                if (updateDto.getUnitUri().equals("http://www.ontology-of-units-of-measure.org/resource/om-2/degreeFahrenheit")) {
+                    column.setValMin(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(32, 68))) /* 0-20 */;
+                    column.setValMax(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(67, 104)).add(column.getValMin())) /* 20-40 */;
+                    column.setMean(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(32, 104)));
+                    column.setMedian(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(32, 104)));
+                    column.setStdDev(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 20)));
+                    log.warn("Faked demo statistical values (deg. Fahrenheit) for column with id {}", columnId);
+                } else if (updateDto.getUnitUri().equals("http://www.ontology-of-units-of-measure.org/resource/om-2/degreeCelsius")) {
+                    column.setValMin(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 20))) /* 0-20 */;
+                    column.setValMax(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(20, 40)).add(column.getValMin())) /* 20-40 */;
+                    column.setMean(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 40)));
+                    column.setMedian(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 40)));
+                    column.setStdDev(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 10)));
+                    log.warn("Faked demo statistical values (deg. Celsius) for column with id {}", columnId);
+                }
+            }
+            // FIXME END democode
             try {
                 column.setUnit(semanticService.findUnit(updateDto.getUnitUri()));
                 log.debug("Found unit with uri {} in metadata database", updateDto.getUnitUri());

dbrepo-search-db/init/indices/column.json

@@ -33,6 +33,21 @@
       "d": {
         "type": "integer"
       },
+      "val_min": {
+        "type": "double"
+      },
+      "val_max": {
+        "type": "double"
+      },
+      "mean": {
+        "type": "double"
+      },
+      "median": {
+        "type": "double"
+      },
+      "std_dev": {
+        "type": "double"
+      },
       "database_id": {
         "type": "keyword"
       },

dbrepo-search-service/app/api/routes.py

@@ -153,7 +153,11 @@ def search():
     logging.debug('search request body: %s', req_body)
     search_term = req_body.get("search_term")
     t1 = req_body.get("t1")
+    if not str(t1).isdigit():
+        t1 = None
     t2 = req_body.get("t2")
+    if not str(t2).isdigit():
+        t2 = None
     fieldValuePairs = req_body.get("field_value_pairs")
     response = general_search(search_term, t1, t2, fieldValuePairs)
     return response, 200

dbrepo-search-service/app/opensearch_client.py

@@ -173,31 +173,21 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None):
         )
         response["status"] = 200
         return response
-    if t1 is not None:
-        logging.debug(f"query has start value {t1} present")
-        time_range_query = {
-            "range": {
-                "created": {
-                    "gte": t1,
-                    "lte": t2,
-                }
-            }
-        }
-        queries.append(time_range_query)
-    if t1 is not None and t2 is not None:
-        logging.debug(f"query has start value {t1} and end value {t2} present")
-        time_range_query = {
-            "range": {
-                "created": {
-                    "gte": t1,
-                    "lte": t2,
-                }
-            }
-        }
-        queries.append(time_range_query)
     if fieldValuePairs is not None and len(fieldValuePairs) > 0:
-        logging.debug('query has fieldValuePairs present')
+        logging.debug('query has field_value_pairs present')
         musts = []
+        is_range_open_end = False
+        is_range_open_begin = False
+        is_range_query = False
+        if t1 is not None and t2 is None:
+            is_range_open_begin = True
+            logging.debug(f"query has only start value {t1} present")
+        if t1 is None and t2 is not None:
+            is_range_open_end = True
+            logging.debug(f"query has only end value {t2} present")
+        if t1 is not None and t2 is not None:
+            is_range_query = True
+            logging.debug(f"query has start value {t1} and end value {t2} present")
         for key, value in fieldValuePairs.items():
             if key == "type" and value in searchable_indices:
                 logging.debug("search for specific index: %s", value)
@@ -209,6 +199,44 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None):
                     logging.debug(
                         f"field name {key} starts with index name {index}: flattened field name to {new_field}")
                     key = new_field
+                if is_range_open_end and re.match(f"unit\\.", key):
+                    logging.debug(f"omit key={key} because query type=open end range and key is somewhat unit")
+                    logging.info(f"add match-query for range ),{t2}]")
+                    musts.append({
+                        "range": {
+                            "val_max": {
+                                "lte": t2
+                            }
+                        }
+                    })
+                elif is_range_open_begin and re.match(f"unit\\.", key):
+                    logging.debug(f"omit key={key} because query type=open begin range and key is somewhat unit")
+                    logging.info(f"add match-query for range [{t1},(")
+                    musts.append({
+                        "range": {
+                            "val_min": {
+                                "gte": t1
+                            }
+                        }
+                    })
+                elif is_range_query and re.match(f"unit\\.", key):
+                    logging.debug(f"omit key={key} because query type=full range and key is somewhat unit")
+                    logging.info(f"add match-query for range [{t1},{t2}]")
+                    musts.append({
+                        "range": {
+                            "val_min": {
+                                "gte": t1
+                            }
+                        }
+                    })
+                    musts.append({
+                        "range": {
+                            "val_max": {
+                                "lte": t2
+                            }
+                        }
+                    })
+                else:
                     musts.append({
                         "match": {
                             key: {"query": value, "minimum_should_match": "90%"}

dbrepo-ui/components/search/AdvancedSearch.vue

@@ -180,6 +180,7 @@ export default {
           return
         }
         this.resetAdvancedSearchFields()
+        this.$emit('search-result', [])
         this.loadingFields = true
         SearchService.getFields(newType)
           .then((response) => {