Skip to content
Snippets Groups Projects
Commit 0662da30 authored by Martin Weise's avatar Martin Weise
Browse files

fixed query mapping 

Former-commit-id: 38d4229f
parent 632f16f6
No related branches found
No related tags found
1 merge request!42Fixed the query service tests
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 604 additions and 388 deletions
docker-compose.yml
+ 1
− 2
View file @ 0662da30
......@@ -57,8 +57,7 @@ services:
build: ./fda-gateway-service
image: fda-gateway-service
networks:
fda-public:
ipv4_address: 172.29.0.6
- fda-public
environment:
SPRING_PROFILES_ACTIVE: docker
ports:
......
fda-metadata-db/api/src/main/java/at/tuwien/api/database/query/ExecuteQueryDto.java
+ 2
− 2
View file @ 0662da30
package at.tuwien.api.database.query;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.*;
@Getter
......@@ -10,7 +9,8 @@ import lombok.*;
@NoArgsConstructor
public class ExecuteQueryDto {
@JsonProperty("Query")
private String title;
private String query;
}
fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/query/File.java
+ 33
− 1
View file @ 0662da30
......@@ -2,15 +2,19 @@ package at.tuwien.entities.database.query;
import lombok.*;
import org.hibernate.annotations.GenericGenerator;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import javax.persistence.*;
import java.time.Instant;
@Data
@Entity
@Builder
@AllArgsConstructor
@NoArgsConstructor
@IdClass(FileKey.class)
@EntityListeners(AuditingEntityListener.class)
@javax.persistence.Table(name = "mdb_files")
public class File {
......@@ -26,10 +30,38 @@ public class File {
)
private Long id;
@Id
@EqualsAndHashCode.Include
@ToString.Include
private Long ftid;
@Id
@EqualsAndHashCode.Include
@ToString.Include
private Long fqid;
@Id
@EqualsAndHashCode.Include
@ToString.Include
private Long fdbid;
@Column(name = "ref_id", nullable = false)
private String refId;
@OneToOne(fetch = FetchType.LAZY)
@JoinColumns({
@JoinColumn(name = "fdbid", referencedColumnName = "qdbid", insertable = false, updatable = false),
@JoinColumn(name = "ftid", referencedColumnName = "qtid", insertable = false, updatable = false),
@JoinColumn(name = "fqid", referencedColumnName = "id", insertable = false, updatable = false)
})
@ManyToOne(fetch = FetchType.LAZY)
private Query query;
@Column(nullable = false, updatable = false)
@CreatedDate
private Instant created;
@Column
@LastModifiedDate
private Instant lastModified;
}
fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/query/FileKey.java 0 → 100644
+ 18
− 0
View file @ 0662da30
package at.tuwien.entities.database.query;
import lombok.EqualsAndHashCode;
import java.io.Serializable;
@EqualsAndHashCode
public class FileKey implements Serializable {
private Long id;
private Long fdbid;
private Long ftid;
private Long fqid;
}
fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/query/Query.java
+ 37
− 3
View file @ 0662da30
......@@ -2,16 +2,20 @@ package at.tuwien.entities.database.query;
import lombok.*;
import org.hibernate.annotations.GenericGenerator;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import javax.persistence.*;
import java.time.Instant;
import java.util.List;
@Data
@Entity
@Builder
@AllArgsConstructor
@NoArgsConstructor
@IdClass(QueryKey.class)
@EntityListeners(AuditingEntityListener.class)
@javax.persistence.Table(name = "mdb_queries")
public class Query {
......@@ -27,18 +31,41 @@ public class Query {
)
private Long id;
@Id
@EqualsAndHashCode.Include
@ToString.Include
private Long qtid;
@Id
@EqualsAndHashCode.Include
@ToString.Include
private Long qdbid;
@Column
private String doi;
@Column(nullable = false)
private String title;
@OneToOne(fetch = FetchType.LAZY, cascade = CascadeType.ALL)
private File file;
@Column(nullable = false)
private String query;
@Column(nullable = false)
private String queryNormalized;
@OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL, mappedBy = "query")
private List<File> files;
@Column(name = "deposit_id")
@Column(name = "deposit_id", unique = true)
private Long depositId;
@JoinColumns({
@JoinColumn(name = "qdbid", referencedColumnName = "tdbid", insertable = false, updatable = false),
@JoinColumn(name = "qtid", referencedColumnName = "id", insertable = false, updatable = false)
})
@ManyToOne(fetch = FetchType.LAZY, cascade = CascadeType.MERGE)
private at.tuwien.entities.database.table.Table table;
@Column
private Instant executionTimestamp;
......@@ -48,5 +75,12 @@ public class Query {
@Column
private Long resultNumber;
@Column(nullable = false, updatable = false)
@CreatedDate
private Instant created;
@Column
@LastModifiedDate
private Instant lastModified;
}
fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/query/QueryKey.java 0 → 100644
+ 16
− 0
View file @ 0662da30
package at.tuwien.entities.database.query;
import lombok.EqualsAndHashCode;
import java.io.Serializable;
@EqualsAndHashCode
public class QueryKey implements Serializable {
private Long id;
private Long qdbid;
private Long qtid;
}
fda-metadata-db/entities/src/main/java/at/tuwien/entities/database/table/Table.java
+ 4
− 0
View file @ 0662da30
package at.tuwien.entities.database.table;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.query.Query;
import at.tuwien.entities.database.table.columns.TableColumn;
import lombok.*;
import org.hibernate.annotations.GenericGenerator;
......@@ -62,6 +63,9 @@ public class Table {
@CreatedDate
private Instant created;
@OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.MERGE, mappedBy = "table")
private List<Query> queries;
@Column
@LastModifiedDate
private Instant lastModified;
......
fda-metadata-db/initi.sh
+ 31
− 25
View file @ 0662da30
......@@ -160,23 +160,6 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
NO MAXVALUE
CACHE 1;
CREATE TABLE IF NOT EXISTS mdb_queries (
ID bigint NOT NULL DEFAULT nextval('mdb_queries_seq'),
execution_timestamp timestamp without time zone NOT NULL,
deposit_id bigint NULL UNIQUE,
file_id bigint NULL,
title character varying(255) NOT NULL,
doi character varying(255),
query TEXT NOT NULL,
query_normalized TEXT NOT NULL,
query_hash character varying(255) NULL,
result_hash character varying(255) NULL,
result_number bigint NULL,
created timestamp without time zone NOT NULL,
last_modified timestamp without time zone,
PRIMARY KEY(ID)
);
CREATE TABLE IF NOT EXISTS mdb_DATABASES (
ID bigint PRIMARY KEY DEFAULT nextval('mdb_databases_seq'),
container_id bigint REFERENCES mdb_CONTAINER(id),
......@@ -211,6 +194,37 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
PRIMARY KEY(tDBID,ID)
);
CREATE TABLE IF NOT EXISTS mdb_queries (
ID bigint NOT NULL DEFAULT nextval('mdb_queries_seq'),
execution_timestamp timestamp without time zone,
deposit_id bigint NULL UNIQUE,
qtid bigint NOT NULL,
qdbid bigint NOT NULL,
title character varying(255) NOT NULL,
doi character varying(255),
query TEXT NOT NULL,
query_normalized TEXT NOT NULL,
query_hash character varying(255) NULL,
result_hash character varying(255) NULL,
result_number bigint NULL,
created timestamp without time zone NOT NULL,
last_modified timestamp without time zone,
FOREIGN KEY (qdbid, qtid) REFERENCES mdb_TABLES(tDBID, ID),
PRIMARY KEY(qdbid, qtid, ID)
);
CREATE TABLE IF NOT EXISTS mdb_files (
id bigint DEFAULT nextval('mdb_files_seq'),
ftid bigint NOT NULL,
fdbid bigint NOT NULL,
fqid bigint NOT NULL,
ref_id varchar(255) NOT NULL,
created timestamp without time zone NOT NULL,
last_modified timestamp without time zone,
FOREIGN KEY (fdbid, ftid, fqid) REFERENCES mdb_queries(qdbid, qtid, ID),
PRIMARY KEY (fdbid, ftid, fqid, id)
);
CREATE TABLE IF NOT EXISTS mdb_COLUMNS (
ID bigint DEFAULT nextval('mdb_columns_seq'),
cDBID bigint,
......@@ -231,14 +245,6 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-E
PRIMARY KEY(cDBID, tID, ID)
);
CREATE TABLE IF NOT EXISTS mdb_files (
id bigint DEFAULT nextval('mdb_files_seq'),
ref_id varchar(255) NOT NULL,
query_id bigint NOT NULL,
FOREIGN KEY (query_id) REFERENCES mdb_queries(id),
PRIMARY KEY (id)
);
CREATE TABLE IF NOT EXISTS mdb_COLUMNS_ENUMS (
ID bigint DEFAULT nextval('mdb_columns_enum_seq'),
eDBID bigint,
......
fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryEndpoint.java
+ 27
− 47
View file @ 0662da30
package at.tuwien.endpoint;
import at.tuwien.api.database.query.ExecuteQueryDto;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.api.database.query.QueryDto;
import at.tuwien.entities.database.query.Query;
import at.tuwien.exception.*;
import at.tuwien.mapper.QueryMapper;
import at.tuwien.service.QueryService;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.ApiResponses;
import net.sf.jsqlparser.JSQLParserException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;
import java.sql.SQLException;
import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/api/database/{id}")
public class QueryEndpoint {
private final QueryMapper queryMapper;
private final QueryService queryService;
@Autowired
public QueryEndpoint(QueryService queryService) {
public QueryEndpoint(QueryMapper queryMapper, QueryService queryService) {
this.queryMapper = queryMapper;
this.queryService = queryService;
}
@Transactional
@PutMapping("/query")
@ApiOperation(value = "executes a query")
@ApiResponses(value = {
@ApiResponse(code = 200, message = "Executed the query, Saved it and return the results"),
@GetMapping("/{queryId}")
@ApiOperation(value = "Find a query", notes = "Find a query")
@ApiResponses({
@ApiResponse(code = 200, message = "All queries are listed."),
@ApiResponse(code = 400, message = "Problem with reading the stored queries."),
@ApiResponse(code = 404, message = "The database does not exist."),
@ApiResponse(code = 405, message = "The container is not running."),
@ApiResponse(code = 409, message = "The container image is not supported."),})
public ResponseEntity<QueryResultDto> execute(@PathVariable Long id,
@RequestBody ExecuteQueryDto data)
throws DatabaseNotFoundException, ImageNotSupportedException, SQLException,
JSQLParserException, QueryStoreException, DatabaseConnectionException {
return ResponseEntity.ok(queryService.execute(id, data));
})
public ResponseEntity<QueryDto> find(@PathVariable Long id, @PathVariable Long queryId)
throws QueryNotFoundException {
return ResponseEntity.ok(queryMapper.queryToQueryDto(queryService.findById(queryId)));
}
@Transactional
@PutMapping("/save")
@ApiOperation(value = "saves a query without execution")
@ApiResponses(value = {
@ApiResponse(code = 200, message = "Executed the query, Saved it and return the results"),
@GetMapping
@ApiOperation(value = "List all queries", notes = "Lists all already executed queries")
@ApiResponses({
@ApiResponse(code = 200, message = "All queries are listed."),
@ApiResponse(code = 400, message = "Problem with reading the stored query."),
@ApiResponse(code = 404, message = "The database does not exist."),
@ApiResponse(code = 405, message = "The container is not running."),
@ApiResponse(code = 409, message = "The container image is not supported."),})
public ResponseEntity<QueryResultDto> save(@PathVariable Long id,
@RequestBody ExecuteQueryDto data)
throws DatabaseNotFoundException, ImageNotSupportedException, JSQLParserException, QueryStoreException,
DatabaseConnectionException, QueryMalformedException {
return ResponseEntity.ok(queryService.save(id, data));
}
@Transactional
@GetMapping("/query/{queryId}")
@ApiOperation(value = "re-executes a query")
@ApiResponses(value = {
@ApiResponse(code = 200, message = "Re-Execute a saved query and return the results"),
@ApiResponse(code = 404, message = "The database does not exist."),
@ApiResponse(code = 405, message = "The container is not running."),
@ApiResponse(code = 409, message = "The container image is not supported."),})
public ResponseEntity<QueryResultDto> reexecute(@PathVariable Long id,
@PathVariable Long queryId,
@RequestParam(name = "page", required = false) Integer page,
@RequestParam(name = "size", required = false) Integer size)
throws DatabaseNotFoundException, ImageNotSupportedException, DatabaseConnectionException,
QueryStoreException, QueryMalformedException {
return ResponseEntity.ok(queryService.reexecute(id, queryId, page, size));
})
public ResponseEntity<List<QueryDto>> findAll(@PathVariable Long id) {
final List<Query> queries = queryService.findAll(id);
return ResponseEntity.ok(queries.stream()
.map(queryMapper::queryToQueryDto)
.collect(Collectors.toList()));
}
......
fda-query-service/rest-service/src/main/java/at/tuwien/endpoint/QueryStoreEndpoint.java
+ 68
− 23
View file @ 0662da30
package at.tuwien.endpoint;
import at.tuwien.api.database.query.ExecuteQueryDto;
import at.tuwien.api.database.query.QueryDto;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.entities.database.query.Query;
......@@ -18,7 +19,7 @@ import java.util.List;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/api/database/{id}/querystore")
@RequestMapping("/api/database/{id}/store")
public class QueryStoreEndpoint {
private QueryStoreService querystoreService;
......@@ -30,21 +31,65 @@ public class QueryStoreEndpoint {
this.queryMapper = queryMapper;
}
@GetMapping("/{queryId}")
@ApiOperation(value = "Find a query", notes = "Find a query")
@ApiResponses({
@ApiResponse(code = 200, message = "All queries are listed."),
@ApiResponse(code = 400, message = "Problem with reading the stored queries."),
@PostMapping
@ApiOperation(value = "Creates the query Store")
@ApiResponses(value = {
@ApiResponse(code = 201, message = "Created the querystore successfully"),
@ApiResponse(code = 404, message = "The database does not exist."),
})
public ResponseEntity<QueryDto> find(@PathVariable Long id,
@PathVariable Long queryId)
@ApiResponse(code = 405, message = "The container is not running."),
@ApiResponse(code = 409, message = "The container image is not supported."),})
public ResponseEntity<?> create(@PathVariable Long id) throws ImageNotSupportedException, DatabaseNotFoundException,
QueryStoreException, DatabaseConnectionException {
querystoreService.create(id);
return ResponseEntity.status(HttpStatus.CREATED)
.build();
}
@PutMapping("/execute")
@ApiOperation(value = "executes a query")
@ApiResponses(value = {
@ApiResponse(code = 200, message = "Executed the query, Saved it and return the results"),
@ApiResponse(code = 404, message = "The database does not exist."),
@ApiResponse(code = 405, message = "The container is not running."),
@ApiResponse(code = 409, message = "The container image is not supported."),})
public ResponseEntity<QueryResultDto> execute(@PathVariable Long id,
@RequestBody ExecuteQueryDto data)
throws QueryStoreException, DatabaseConnectionException, QueryMalformedException, DatabaseNotFoundException,
ImageNotSupportedException {
return ResponseEntity.ok(querystoreService.execute(id, data));
}
@PostMapping("/save")
@ApiOperation(value = "saves a query without execution")
@ApiResponses(value = {
@ApiResponse(code = 200, message = "Executed the query, Saved it and return the results"),
@ApiResponse(code = 404, message = "The database does not exist."),
@ApiResponse(code = 405, message = "The container is not running."),
@ApiResponse(code = 409, message = "The container image is not supported."),})
public ResponseEntity<QueryResultDto> save(@PathVariable Long id,
@RequestBody ExecuteQueryDto data)
throws DatabaseNotFoundException, ImageNotSupportedException, QueryStoreException,
DatabaseConnectionException, QueryMalformedException {
return ResponseEntity.ok(querystoreService.save(id, data));
}
@PutMapping("/execute/{queryId}")
@ApiOperation(value = "re-executes a query")
@ApiResponses(value = {
@ApiResponse(code = 200, message = "Re-Execute a saved query and return the results"),
@ApiResponse(code = 404, message = "The database does not exist."),
@ApiResponse(code = 405, message = "The container is not running."),
@ApiResponse(code = 409, message = "The container image is not supported."),})
public ResponseEntity<QueryResultDto> reexecute(@PathVariable Long id,
@PathVariable Long queryId,
@RequestParam(name = "page", required = false) Integer page,
@RequestParam(name = "size", required = false) Integer size)
throws DatabaseNotFoundException, ImageNotSupportedException, DatabaseConnectionException,
QueryStoreException {
return ResponseEntity.ok(querystoreService.findOne(id, queryId));
QueryStoreException, QueryMalformedException {
return ResponseEntity.ok(querystoreService.reexecute(id, queryId, page, size));
}
@GetMapping
@GetMapping("/query")
@ApiOperation(value = "List all queries", notes = "Lists all already executed queries")
@ApiResponses({
@ApiResponse(code = 200, message = "All queries are listed."),
......@@ -59,17 +104,17 @@ public class QueryStoreEndpoint {
.collect(Collectors.toList()));
}
@PostMapping
@ApiOperation(value = "Creates the query Store")
@ApiResponses(value = {
@ApiResponse(code = 201, message = "Created the Querystore successfully"),
@GetMapping("/query/{queryId}")
@ApiOperation(value = "Find a query", notes = "Find a query")
@ApiResponses({
@ApiResponse(code = 200, message = "All queries are listed."),
@ApiResponse(code = 400, message = "Problem with reading the stored queries."),
@ApiResponse(code = 404, message = "The database does not exist."),
@ApiResponse(code = 405, message = "The container is not running."),
@ApiResponse(code = 409, message = "The container image is not supported."),})
public ResponseEntity<?> create(@PathVariable Long id) throws ImageNotSupportedException, DatabaseNotFoundException,
QueryStoreException, DatabaseConnectionException {
querystoreService.create(id);
return ResponseEntity.status(HttpStatus.CREATED)
.build();
})
public ResponseEntity<QueryDto> find(@PathVariable Long id,
@PathVariable Long queryId)
throws DatabaseNotFoundException, ImageNotSupportedException, DatabaseConnectionException,
QueryStoreException {
return ResponseEntity.ok(querystoreService.findOne(id, queryId));
}
}
fda-query-service/rest-service/src/main/resources/application.yml
+ 1
− 1
View file @ 0662da30
......@@ -9,7 +9,7 @@ spring:
show-sql: true
database-platform: org.hibernate.dialect.PostgreSQLDialect
hibernate:
ddl-auto: update
ddl-auto: validate
open-in-view: false
application:
name: fda-query-service
......
fda-query-service/services/src/main/java/at/tuwien/exception/QueryNotFoundException.java 0 → 100644
+ 21
− 0
View file @ 0662da30
package at.tuwien.exception;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseStatus(code = HttpStatus.NOT_FOUND)
public class QueryNotFoundException extends Exception {
public QueryNotFoundException(String msg) {
super(msg);
}
public QueryNotFoundException(String msg, Throwable thr) {
super(msg, thr);
}
public QueryNotFoundException(Throwable thr) {
super(thr);
}
}
fda-query-service/services/src/main/java/at/tuwien/mapper/QueryMapper.java
+ 9
− 2
View file @ 0662da30
......@@ -27,12 +27,13 @@ public interface QueryMapper {
Query queryDtotoQuery(QueryDto data);
@Mappings({
@Mapping(source = "data.executionTimestamp", target = "executionTimestamp", qualifiedByName = "timezoned")
@Mapping(source = "executionTimestamp", target = "executionTimestamp", qualifiedByName = "timezoned")
})
QueryDto queryToQueryDto(Query data);
@Mappings({
@Mapping(source = "query", target = "query")
@Mapping(source = "query", target = "query"),
@Mapping(source = "query", target = "queryNormalized")
})
QueryDto executeQueryDtoToQueryDto(ExecuteQueryDto data);
......@@ -76,6 +77,9 @@ public interface QueryMapper {
}
default Instant objectToInstant(Object data) {
if (data == null) {
return null;
}
final DateTimeFormatter formatter = DateTimeFormatter
.ofPattern("yyyy-MM-dd HH:mm:ss.S")
.withZone(ZoneId.systemDefault());
......@@ -84,6 +88,9 @@ public interface QueryMapper {
@Named("timezoned")
default Instant objectToInstantWithTimezone(Object data) {
if (data == null) {
return null;
}
final DateTimeFormatter formatter = DateTimeFormatter
.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'")
.withZone(ZoneId.systemDefault());
......
fda-query-service/services/src/main/java/at/tuwien/repository/ContainerRepository.java → fda-query-service/services/src/main/java/at/tuwien/repository/jpa/ContainerRepository.java
+ 1
− 1
View file @ 0662da30
package at.tuwien.repository;
package at.tuwien.repository.jpa;
import at.tuwien.entities.container.Container;
import org.springframework.data.jpa.repository.JpaRepository;
......
fda-query-service/services/src/main/java/at/tuwien/repository/ImageRepository.java → fda-query-service/services/src/main/java/at/tuwien/repository/jpa/ImageRepository.java
+ 1
− 1
View file @ 0662da30
package at.tuwien.repository;
package at.tuwien.repository.jpa;
import at.tuwien.entities.container.image.ContainerImage;
import org.springframework.data.jpa.repository.JpaRepository;
......
fda-query-service/services/src/main/java/at/tuwien/repository/jpa/QueryRepository.java 0 → 100644
+ 16
− 0
View file @ 0662da30
package at.tuwien.repository.jpa;
import at.tuwien.entities.database.query.Query;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface QueryRepository extends JpaRepository<Query, Long> {
@org.springframework.data.jpa.repository.Query(value = "select q from Query q where q.table.database.id = :id")
List<Query> findAllByDatabaseId(@Param("id") Long id);
}
fda-query-service/services/src/main/java/at/tuwien/repository/TableRepository.java → fda-query-service/services/src/main/java/at/tuwien/repository/jpa/TableRepository.java
+ 1
− 1
View file @ 0662da30
package at.tuwien.repository;
package at.tuwien.repository.jpa;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.table.Table;
......
fda-query-service/services/src/main/java/at/tuwien/service/JdbcConnector.java
+ 2
− 15
View file @ 0662da30
package at.tuwien.service;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.api.database.table.TableCreateDto;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.query.Query;
import at.tuwien.exception.DatabaseConnectionException;
import at.tuwien.exception.DatabaseNotFoundException;
import at.tuwien.exception.ImageNotSupportedException;
import at.tuwien.exception.QueryMalformedException;
import at.tuwien.mapper.ImageMapper;
import at.tuwien.mapper.QueryMapper;
import lombok.extern.log4j.Log4j2;
import org.jooq.*;
import org.jooq.Record;
import org.jooq.impl.DSL;
import org.springframework.beans.factory.annotation.Autowired;
import java.sql.*;
import java.util.List;
import java.util.Properties;
import static org.jooq.impl.DSL.*;
@Log4j2
public abstract class JdbcConnector {
private final ImageMapper imageMapper;
private final QueryMapper queryMapper;
@Autowired
protected JdbcConnector(ImageMapper imageMapper, QueryMapper queryMapper) {
protected JdbcConnector(ImageMapper imageMapper) {
this.imageMapper = imageMapper;
this.queryMapper = queryMapper;
}
protected DSLContext open(Database database) throws SQLException, ImageNotSupportedException {
final String url = "jdbc:" + database.getContainer().getImage().getJdbcMethod() + "://" + database.getContainer().getInternalName() + "/" + database.getInternalName();
log.info("Attempt to connect to '{}'", url);
log.trace("Attempt to connect to '{}'", url);
final Connection connection = DriverManager.getConnection(url, imageMapper.containerImageToProperties(database.getContainer().getImage()));
return DSL.using(connection, SQLDialect.valueOf(database.getContainer().getImage().getDialect()));
}
......
fda-query-service/services/src/main/java/at/tuwien/service/QueryService.java
+ 14
− 246
View file @ 0662da30
package at.tuwien.service;
import at.tuwien.api.database.query.ExecuteQueryDto;
import at.tuwien.api.database.query.QueryDto;
import at.tuwien.api.database.query.QueryResultDto;
import at.tuwien.entities.database.Database;
import at.tuwien.entities.database.query.Query;
import at.tuwien.entities.database.table.Table;
import at.tuwien.entities.database.table.columns.TableColumn;
import at.tuwien.exception.*;
import at.tuwien.mapper.ImageMapper;
import at.tuwien.mapper.QueryMapper;
import at.tuwien.repository.jpa.DatabaseRepository;
import at.tuwien.repository.jpa.QueryRepository;
import lombok.extern.log4j.Log4j2;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.Expression;
import net.sf.jsqlparser.parser.CCJSqlParserManager;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.*;
import org.jooq.DSLContext;
import org.jooq.Record;
import org.jooq.Result;
import org.jooq.ResultQuery;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.io.StringReader;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
@Log4j2
@Service
public class QueryService extends JdbcConnector {
private final DatabaseRepository databaseRepository;
private final QueryStoreService queryStoreService;
private final QueryMapper queryMapper;
private final QueryRepository queryRepository;
@Autowired
public QueryService(ImageMapper imageMapper, QueryMapper queryMapper, DatabaseRepository databaseRepository, QueryStoreService queryStoreService) {
super(imageMapper, queryMapper);
this.databaseRepository = databaseRepository;
this.queryStoreService = queryStoreService;
this.queryMapper = queryMapper;
public QueryService(ImageMapper imageMapper, QueryRepository queryRepository) {
super(imageMapper);
this.queryRepository = queryRepository;
}
@Transactional
public QueryResultDto execute(Long id, ExecuteQueryDto data) throws ImageNotSupportedException, DatabaseNotFoundException,
JSQLParserException, SQLException, QueryStoreException, DatabaseConnectionException {
final Database database = findDatabase(id);
if (database.getContainer().getImage().getDialect().equals("MARIADB")) {
if (!queryStoreService.exists(database)) {
queryStoreService.create(id);
}
}
final DSLContext context = open(database);
final QueryDto query = queryMapper.executeQueryDtoToQueryDto(data);
//TODO Fix that import
final StringBuilder parsedQuery = new StringBuilder();
final String q = parse(query, database).getQuery();
if (q.charAt(q.length() - 1) == ';') {
parsedQuery.append(q.substring(0, q.length() - 2));
} else {
parsedQuery.append(q);
}
parsedQuery.append(";");
final ResultQuery<Record> resultQuery = context.resultQuery(parsedQuery.toString());
final Result<Record> result = resultQuery.fetch();
final QueryResultDto queryResultDto = queryMapper.recordListToQueryResultDto(result, query.getId());
log.trace("Result of the query is: \n {}", result.toString());
// Save the query in the store
final QueryDto out = queryStoreService.saveQuery(database, query, queryResultDto);
queryResultDto.setId(out.getId());
log.info("Saved query {}", out);
return queryResultDto;
}
private QueryDto parse(QueryDto query, Database database) throws JSQLParserException {
query.setExecutionTimestamp(query.getExecutionTimestamp());
CCJSqlParserManager parserRealSql = new CCJSqlParserManager();
Statement statement = parserRealSql.parse(new StringReader(query.getQuery()));
log.debug("Given query {}", query.getQuery());
if (statement instanceof Select) {
Select selectStatement = (Select) statement;
PlainSelect ps = (PlainSelect) selectStatement.getSelectBody();
List<SelectItem> selectItems = ps.getSelectItems();
//Parse all tables
List<FromItem> fromItems = new ArrayList<>();
fromItems.add(ps.getFromItem());
if (ps.getJoins() != null && ps.getJoins().size() > 0) {
for (Join j : ps.getJoins()) {
if (j.getRightItem() != null) {
fromItems.add(j.getRightItem());
}
}
}
//Checking if all tables exist
List<TableColumn> allColumns = new ArrayList<>();
for (FromItem f : fromItems) {
boolean i = false;
log.debug("from item iterated through: {}", f);
for (Table t : database.getTables()) {
if (f.toString().equals(t.getInternalName()) || f.toString().equals(t.getName())) {
allColumns.addAll(t.getColumns());
i = false;
break;
}
i = true;
}
if (i) {
throw new JSQLParserException("Table " + f.toString() + " does not exist");
}
}
//Checking if all columns exist
for (SelectItem s : selectItems) {
String select = s.toString();
if (select.trim().equals("*")) {
log.debug("Please do not use * to query data");
continue;
}
// ignore prefixes
if (select.contains(".")) {
log.debug(select);
select = select.split("\\.")[1];
}
boolean i = false;
for (TableColumn tc : allColumns) {
log.debug("{},{},{}", tc.getInternalName(), tc.getName(), s);
if (select.equals(tc.getInternalName()) || select.toString().equals(tc.getName())) {
i = false;
break;
}
i = true;
}
if (i) {
throw new JSQLParserException("Column " + s.toString() + " does not exist");
}
}
//TODO Future work
if (ps.getWhere() != null) {
Expression where = ps.getWhere();
log.debug("Where clause: {}", where);
}
return query;
} else {
throw new JSQLParserException("SQL Query is not a SELECT statement - please only use SELECT statements");
}
public List<Query> findAll(Long databaseId) {
return queryRepository.findAllByDatabaseId(databaseId);
}
@Transactional
public Database findDatabase(Long id) throws DatabaseNotFoundException {
final Optional<Database> database = databaseRepository.findById(id);
if (database.isEmpty()) {
log.error("no database with this id found in metadata database");
throw new DatabaseNotFoundException("database not found in metadata database");
}
return database.get();
public Query findById(Long queryId) throws QueryNotFoundException {
final Optional<Query> query = queryRepository.findById(queryId);
if (query.isEmpty()) {
log.error("Query with id {} was not found to metadata database", queryId);
throw new QueryNotFoundException("Query was not found to metadata database");
}
/**
* Re-executes a query with given id in a database by given id, returns a result of size and offset
*
* @param databaseId The database id
* @param queryId The query id
* @param page The offset
* @param size The size
* @return The result set
* @throws DatabaseNotFoundException The database was not found in the metadata database
* @throws SQLException The mapped query is not valid SQL
* @throws ImageNotSupportedException The image is not supported
* @throws DatabaseConnectionException The remote container is not available right now
* @throws QueryStoreException There was an error with the query store in the remote container
*/
public QueryResultDto reexecute(Long databaseId, Long queryId, Integer page, Integer size)
throws DatabaseNotFoundException, ImageNotSupportedException,
QueryStoreException, QueryMalformedException, DatabaseConnectionException {
log.info("re-execute query with the id {}", queryId);
final DSLContext context;
try {
context = open(findDatabase(databaseId));
} catch (SQLException e) {
throw new QueryStoreException("Could not establish connection to query store", e);
}
final QueryDto savedQuery = queryStoreService.findOne(databaseId, queryId);
final StringBuilder query = new StringBuilder();
query.append("SELECT * FROM (");
final String q = savedQuery.getQuery();
if (q.toLowerCase(Locale.ROOT).contains("where")) {
String[] split = q.toLowerCase(Locale.ROOT).split("where");
if (split.length > 2) {
//TODO FIX SUBQUERIES WITH MULTIPLE Wheres
throw new QueryMalformedException("Query Contains Subqueries, this will be supported in a future version");
} else {
query.append(split[0])
.append(" FOR SYSTEM_TIME AS OF TIMESTAMP'")
.append(Timestamp.valueOf(savedQuery.getExecutionTimestamp().toString()))
.append("' WHERE")
.append(split[1])
.append(") as tab");
}
} else {
query.append(q)
.append(" FOR SYSTEM_TIME AS OF TIMESTAMP'")
.append(Timestamp.valueOf(savedQuery.getExecutionTimestamp().toString()))
.append("') as tab");
}
if (page != null && size != null) {
page = Math.abs(page);
size = Math.abs(size);
query.append(" LIMIT ")
.append(size)
.append(" OFFSET ")
.append(page * size);
}
query.append(";");
final Result<org.jooq.Record> result = context.resultQuery(query.toString())
.fetch();
log.debug("query string {}", query.toString());
log.trace("query result \n {}", result.toString());
return queryMapper.recordListToQueryResultDto(result, queryId);
}
@Transactional
public QueryResultDto save(Long id, ExecuteQueryDto data) throws ImageNotSupportedException,
DatabaseNotFoundException, QueryStoreException, JSQLParserException, DatabaseConnectionException,
QueryMalformedException {
final Database database = findDatabase(id);
if (database.getContainer().getImage().getDialect().equals("MARIADB")) {
if (!queryStoreService.exists(database)) {
queryStoreService.create(id);
}
}
final QueryDto query = queryMapper.executeQueryDtoToQueryDto(data);
final DSLContext context;
try {
context = open(database);
} catch (SQLException e) {
throw new QueryMalformedException("Could not connect to the remote container database", e);
}
final StringBuilder parsedQuery = new StringBuilder();
final String q = parse(query, database).getQuery();
if (q.charAt(q.length() - 1) == ';') {
parsedQuery.append(q.substring(0, q.length() - 2));
} else {
parsedQuery.append(q);
}
parsedQuery.append(";");
final ResultQuery<Record> resultQuery = context.resultQuery(parsedQuery.toString());
final Result<Record> result = resultQuery.fetch();
final QueryResultDto queryResultDto = queryMapper.recordListToQueryResultDto(result, query.getId());
log.trace("Result of the query is: \n {}", result.toString());
// Save the query in the store
final QueryDto out = queryStoreService.saveQuery(database, query, queryResultDto);
queryResultDto.setId(out.getId());
log.debug("Save query {}", out);
// return queryStoreService.findLast(database.getId(), query); // FIXME mw: why query last entry when we set it in the line above?
return queryResultDto;
return query.get();
}
}
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment