Skip to content
Snippets Groups Projects
Unverified Commit 7ed8898e authored by Martin Weise's avatar Martin Weise
Browse files

Added range query search

parent ae62095e
No related branches found
No related tags found
4 merge requests!231CI: Remove build for log-service,!228Better error message handling in the frontend,!223Release of version 1.4.0,!215Resolve "Fix the unit independent search"
Show whitespace changes
Inline Side-by-side
Showing
with 144 additions and 46 deletions
dbrepo-metadata-db/setup-schema.sql
+ 7
− 18
View file @ 7ed8898e
......@@ -160,6 +160,11 @@ CREATE TABLE IF NOT EXISTS `mdb_columns`
d INT,
auto_generated BOOLEAN DEFAULT false,
is_null_allowed BOOLEAN NOT NULL DEFAULT true,
val_min NUMERIC NULL,
val_max NUMERIC NULL,
mean NUMERIC NULL,
median NUMERIC NULL,
std_dev Numeric NULL,
created timestamp NOT NULL DEFAULT NOW(),
last_modified timestamp,
FOREIGN KEY (tID) REFERENCES mdb_tables (ID),
......@@ -195,23 +200,6 @@ CREATE TABLE IF NOT EXISTS `mdb_columns_nom`
PRIMARY KEY (tID, cID)
) WITH SYSTEM VERSIONING;
CREATE TABLE IF NOT EXISTS `mdb_columns_num`
(
tID bigint,
cID bigint,
SIunit TEXT,
MaxVal NUMERIC,
MinVal NUMERIC,
Mean NUMERIC,
Median NUMERIC,
Sd Numeric,
-- Histogram INTEGER[],
last_modified timestamp,
created timestamp NOT NULL DEFAULT NOW(),
FOREIGN KEY (tID, cID) REFERENCES mdb_columns (tID, ID),
PRIMARY KEY (tID, cID)
) WITH SYSTEM VERSIONING;
CREATE TABLE IF NOT EXISTS `mdb_columns_cat`
(
tID bigint,
......@@ -546,7 +534,8 @@ VALUES (1, '%Y-%c-%d %H:%i:%S.%f', 'yyyy-MM-dd HH:mm:ss.SSSSSS', '2022-01-30 13:
INSERT INTO `mdb_ontologies` (prefix, uri, uri_pattern, sparql_endpoint, rdf_path)
VALUES ('om', 'http://www.ontology-of-units-of-measure.org/resource/om-2/',
'http://www.ontology-of-units-of-measure.org/resource/om-2/.*', null, 'rdf/om-2.0.rdf'),
('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql', null),
('wd', 'http://www.wikidata.org/', 'http://www.wikidata.org/entity/.*', 'https://query.wikidata.org/sparql',
null),
('mo', 'http://purl.org/ontology/mo/', 'http://purl.org/ontology/mo/.*', null, null),
('dc', 'http://purl.org/dc/elements/1.1/', null, null, null),
('xsd', 'http://www.w3.org/2001/XMLSchema#', null, null, null),
......
dbrepo-metadata-service/api/src/main/java/at/tuwien/api/database/table/columns/ColumnDto.java
+ 22
− 0
View file @ 7ed8898e
......@@ -6,6 +6,7 @@ import at.tuwien.api.database.table.columns.concepts.UnitDto;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.persistence.Column;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.*;
......@@ -15,6 +16,7 @@ import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
import java.math.BigDecimal;
import java.util.List;
@Getter
......@@ -91,6 +93,26 @@ public class ColumnDto {
@Field(name = "d", type = FieldType.Integer)
private Integer d;
@Schema(example = "0")
@Field(name = "val_min", type = FieldType.Double)
private BigDecimal valMin;
@Schema(example = "100")
@Field(name = "val_max", type = FieldType.Double)
private BigDecimal valMax;
@Schema(example = "45.4")
@Field(name = "mean", type = FieldType.Double)
private BigDecimal mean;
@Schema(example = "51")
@Field(name = "median", type = FieldType.Double)
private BigDecimal median;
@Schema(example = "5.32")
@Field(name = "std_dev", type = FieldType.Double)
private BigDecimal stdDev;
@Field(name = "concept", includeInParent = true, type = FieldType.Nested)
private ConceptDto concept;
......
dbrepo-metadata-service/entities/src/main/java/at/tuwien/entities/database/table/columns/TableColumn.java
+ 16
− 0
View file @ 7ed8898e
......@@ -12,6 +12,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import jakarta.persistence.*;
import java.math.BigDecimal;
import java.time.Instant;
import java.util.List;
......@@ -116,6 +117,21 @@ public class TableColumn implements Comparable<TableColumn> {
@Column
private Integer d;
@Column(name = "val_min")
private BigDecimal valMin;
@Column(name = "val_max")
private BigDecimal valMax;
@Column
private BigDecimal mean;
@Column
private BigDecimal median;
@Column(name = "std_dev")
private BigDecimal stdDev;
@LastModifiedDate
@Column(columnDefinition = "TIMESTAMP")
private Instant lastModified;
......
dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/TableServiceImpl.java
+ 23
− 0
View file @ 7ed8898e
......@@ -20,12 +20,14 @@ import at.tuwien.service.SemanticService;
import at.tuwien.service.TableService;
import at.tuwien.service.UserService;
import at.tuwien.utils.UserUtil;
import ch.qos.logback.core.testUtil.RandomUtil;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.math.BigDecimal;
import java.security.Principal;
import java.sql.Connection;
import java.sql.PreparedStatement;
......@@ -33,6 +35,8 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
@Log4j2
@Service
......@@ -244,6 +248,25 @@ public class TableServiceImpl extends HibernateConnector implements TableService
final TableColumn column = findColumn(table, columnId);
/* assign */
if (updateDto.getUnitUri() != null) {
// FIXME BEGIN democode
if (updateDto.getConceptUri().equals("http://www.wikidata.org/entity/Q11466")) {
if (updateDto.getUnitUri().equals("http://www.ontology-of-units-of-measure.org/resource/om-2/degreeFahrenheit")) {
column.setValMin(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(32, 68))) /* 0-20 */;
column.setValMax(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(67, 104)).add(column.getValMin())) /* 20-40 */;
column.setMean(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(32, 104)));
column.setMedian(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(32, 104)));
column.setStdDev(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 20)));
log.warn("Faked demo statistical values (deg. Fahrenheit) for column with id {}", columnId);
} else if (updateDto.getUnitUri().equals("http://www.ontology-of-units-of-measure.org/resource/om-2/degreeCelsius")) {
column.setValMin(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 20))) /* 0-20 */;
column.setValMax(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(20, 40)).add(column.getValMin())) /* 20-40 */;
column.setMean(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 40)));
column.setMedian(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 40)));
column.setStdDev(BigDecimal.valueOf(ThreadLocalRandom.current().nextDouble(0, 10)));
log.warn("Faked demo statistical values (deg. Celsius) for column with id {}", columnId);
}
}
// FIXME END democode
try {
column.setUnit(semanticService.findUnit(updateDto.getUnitUri()));
log.debug("Found unit with uri {} in metadata database", updateDto.getUnitUri());
......
dbrepo-search-db/init/indices/column.json
+ 15
− 0
View file @ 7ed8898e
......@@ -33,6 +33,21 @@
"d": {
"type": "integer"
},
"val_min": {
"type": "double"
},
"val_max": {
"type": "double"
},
"mean": {
"type": "double"
},
"median": {
"type": "double"
},
"std_dev": {
"type": "double"
},
"database_id": {
"type": "keyword"
},
......
dbrepo-search-service/app/api/routes.py
+ 4
− 0
View file @ 7ed8898e
......@@ -153,7 +153,11 @@ def search():
logging.debug('search request body: %s', req_body)
search_term = req_body.get("search_term")
t1 = req_body.get("t1")
if not str(t1).isdigit():
t1 = None
t2 = req_body.get("t2")
if not str(t2).isdigit():
t2 = None
fieldValuePairs = req_body.get("field_value_pairs")
response = general_search(search_term, t1, t2, fieldValuePairs)
return response, 200
dbrepo-search-service/app/opensearch_client.py
+ 56
− 28
View file @ 7ed8898e
......@@ -173,31 +173,21 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None):
)
response["status"] = 200
return response
if t1 is not None:
logging.debug(f"query has start value {t1} present")
time_range_query = {
"range": {
"created": {
"gte": t1,
"lte": t2,
}
}
}
queries.append(time_range_query)
if t1 is not None and t2 is not None:
logging.debug(f"query has start value {t1} and end value {t2} present")
time_range_query = {
"range": {
"created": {
"gte": t1,
"lte": t2,
}
}
}
queries.append(time_range_query)
if fieldValuePairs is not None and len(fieldValuePairs) > 0:
logging.debug('query has fieldValuePairs present')
logging.debug('query has field_value_pairs present')
musts = []
is_range_open_end = False
is_range_open_begin = False
is_range_query = False
if t1 is not None and t2 is None:
is_range_open_begin = True
logging.debug(f"query has only start value {t1} present")
if t1 is None and t2 is not None:
is_range_open_end = True
logging.debug(f"query has only end value {t2} present")
if t1 is not None and t2 is not None:
is_range_query = True
logging.debug(f"query has start value {t1} and end value {t2} present")
for key, value in fieldValuePairs.items():
if key == "type" and value in searchable_indices:
logging.debug("search for specific index: %s", value)
......@@ -209,6 +199,44 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None):
logging.debug(
f"field name {key} starts with index name {index}: flattened field name to {new_field}")
key = new_field
if is_range_open_end and re.match(f"unit\\.", key):
logging.debug(f"omit key={key} because query type=open end range and key is somewhat unit")
logging.info(f"add match-query for range ),{t2}]")
musts.append({
"range": {
"val_max": {
"lte": t2
}
}
})
elif is_range_open_begin and re.match(f"unit\\.", key):
logging.debug(f"omit key={key} because query type=open begin range and key is somewhat unit")
logging.info(f"add match-query for range [{t1},(")
musts.append({
"range": {
"val_min": {
"gte": t1
}
}
})
elif is_range_query and re.match(f"unit\\.", key):
logging.debug(f"omit key={key} because query type=full range and key is somewhat unit")
logging.info(f"add match-query for range [{t1},{t2}]")
musts.append({
"range": {
"val_min": {
"gte": t1
}
}
})
musts.append({
"range": {
"val_max": {
"lte": t2
}
}
})
else:
musts.append({
"match": {
key: {"query": value, "minimum_should_match": "90%"}
......
dbrepo-ui/components/search/AdvancedSearch.vue
+ 1
− 0
View file @ 7ed8898e
......@@ -180,6 +180,7 @@ export default {
return
}
this.resetAdvancedSearchFields()
this.$emit('search-result', [])
this.loadingFields = true
SearchService.getFields(newType)
.then((response) => {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment