Skip to content
Snippets Groups Projects
Verified Commit 9c716294 authored by Martin Weise's avatar Martin Weise
Browse files

WIP (not working)

parent b1c6a73a
No related branches found
No related tags found
No related merge requests found
Showing
with 245 additions and 8 deletions
......@@ -84,6 +84,12 @@ services:
- ./config/import-realms.sh:/docker-entrypoint-initdb.d/import-realms.sh
- ./config/master-realm.json:/opt/keycloak/data/import/master-realm.json
- ./config/dbrepo-realm.json:/opt/keycloak/data/import/dbrepo-realm.json
<<<<<<< Updated upstream
=======
- ./config/create-event-listener.jar:/opt/bitnami/keycloak/providers/create-event-listener.jar
ports:
- "8080:8080"
>>>>>>> Stashed changes
environment:
KEYCLOAK_ENABLE_HTTPS: "false"
KEYCLOAK_ENABLE_STATISTICS: "true"
......@@ -92,6 +98,9 @@ services:
KEYCLOAK_DATABASE_NAME: "${AUTH_DB_NAME:-keycloak}"
KEYCLOAK_DATABASE_USER: "${AUTH_DB_USERNAME:-keycloak}"
KEYCLOAK_DATABASE_PASSWORD: "${AUTH_DB_PASSWORD:-dbrepo}"
METADATA_SERVICE_ENDPOINT: "${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}/api/user"
SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
SYSTEM_PASSWORD: "${SYSTEM_PASSWORD:-admin}"
healthcheck:
test: curl --head -fsS http://localhost:9000/health/ready
interval: 10s
......@@ -105,11 +114,39 @@ services:
logging:
driver: json-file
<<<<<<< Updated upstream
=======
dbrepo-auth-service-init:
init: true
restart: "no"
container_name: dbrepo-auth-service-init
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.6.3
environment:
AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-admin}
AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-admin}
AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
METADATA_DB: "${METADATA_DB:-dbrepo}"
METADATA_DB_PASSWORD: "${METADATA_DB_PASSWORD:-dbrepo}"
METADATA_USERNAME: "root"
SYSTEM_USERNAME: "${SYSTEM_USERNAME:-admin}"
depends_on:
dbrepo-auth-service:
condition: service_healthy
dbrepo-metadata-db:
condition: service_healthy
logging:
driver: json-file
>>>>>>> Stashed changes
dbrepo-metadata-service:
restart: "no"
container_name: dbrepo-metadata-service
hostname: metadata-service
<<<<<<< Updated upstream
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.4.7
=======
image: registry.datalab.tuwien.ac.at/dbrepo/metadata-service:1.6.3
>>>>>>> Stashed changes
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
......@@ -172,7 +209,11 @@ services:
restart: "no"
container_name: dbrepo-analyse-service
hostname: analyse-service
<<<<<<< Updated upstream
image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.4.7
=======
image: registry.datalab.tuwien.ac.at/dbrepo/analyse-service:1.6.3
>>>>>>> Stashed changes
environment:
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
......@@ -225,7 +266,11 @@ services:
restart: "no"
container_name: dbrepo-search-db
hostname: search-db
<<<<<<< Updated upstream
image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.4.7
=======
image: registry.datalab.tuwien.ac.at/dbrepo/search-db:1.6.3
>>>>>>> Stashed changes
healthcheck:
test: curl -sSL localhost:9200/_plugins/_security/health | jq .status | grep UP
interval: 10s
......@@ -249,7 +294,11 @@ services:
restart: "no"
container_name: dbrepo-search-service
hostname: search-service
<<<<<<< Updated upstream
image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.4.7
=======
image: registry.datalab.tuwien.ac.at/dbrepo/search-service:1.6.3
>>>>>>> Stashed changes
environment:
AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT_SECRET:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
......@@ -292,7 +341,11 @@ services:
restart: "no"
container_name: dbrepo-ui
hostname: ui
<<<<<<< Updated upstream
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.7
=======
image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.6.3
>>>>>>> Stashed changes
environment:
NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}"
NUXT_PUBLIC_API_SERVER: "${BASE_URL:-http://localhost}"
......@@ -360,7 +413,11 @@ services:
init: true
container_name: dbrepo-search-service-init
hostname: search-service-init
<<<<<<< Updated upstream
image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.7
=======
image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.6.3
>>>>>>> Stashed changes
environment:
METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
......@@ -414,7 +471,15 @@ services:
restart: "no"
container_name: dbrepo-dashboard-service
hostname: dashboard-service
<<<<<<< Updated upstream
image: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.4.7
=======
image: registry.datalab.tuwien.ac.at/dbrepo/dashboard-service:1.6.3
ports:
- "3000:3000"
volumes:
- dashboard-service-data:/opt/bitnami/grafana/data
>>>>>>> Stashed changes
environment:
LDAP_ADMIN_USERNAME: "${IDENTITY_SERVICE_ADMIN_USERNAME:-admin}"
LDAP_ADMIN_PASSWORD: "${IDENTITY_SERVICE_ADMIN_PASSWORD:-admin}"
......@@ -435,7 +500,11 @@ services:
init: true
container_name: dbrepo-storage-service-init
hostname: storage-service-init
<<<<<<< Updated upstream
image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.4.7
=======
image: registry.datalab.tuwien.ac.at/dbrepo/storage-service-init:1.6.3
>>>>>>> Stashed changes
environment:
WEED_CLUSTER_SW_MASTER: "${STORAGE_SERVICE_MASTER_ENDPOINT:-storage-service:9333}"
S3_BUCKET: "${S3_BUCKET:-dbrepo}"
......@@ -475,7 +544,11 @@ services:
restart: "no"
container_name: dbrepo-data-service
hostname: data-service
<<<<<<< Updated upstream
image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.4.7
=======
image: registry.datalab.tuwien.ac.at/dbrepo/data-service:1.6.3
>>>>>>> Stashed changes
volumes:
- "${SHARED_VOLUME:-/tmp}:/tmp"
environment:
......
......@@ -32,6 +32,95 @@ stages:
- verify
- scan
<<<<<<< Updated upstream
=======
lint-docker-compose:
image: docker.io/alpine:${ALPINE_VERSION}
stage: lint
variables:
VERSION: 4.45.1
BINARY: yq_linux_amd64
before_script:
- 'apk --no-cache add bash wget'
- 'wget https://github.com/mikefarah/yq/releases/download/v${VERSION}/${BINARY} -O /usr/bin/yq && chmod +x /usr/bin/yq'
script:
- "bash .scripts/check-compose.sh"
- "diff <(yq '.volumes' docker-compose.yml) <(yq '.volumes' .docker/docker-compose.yml)"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-analyse-service'"
- "IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-auth-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-auth-service'"
- "IGNORE_VOLUMES=1 IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-auth-service-init'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-broker-service'"
- "IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-dashboard-service'"
- "bash .scripts/check-service.sh 'dbrepo-data-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-data-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-gateway-service'"
- "IGNORE_VOLUMES=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-identity-service'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-metadata-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-metadata-service'"
- "IGNORE_VOLUMES=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-metric-db'"
- "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-db'"
- "IGNORE_IMAGE=1 IGNORE_PORTS=1 bash .scripts/check-service.sh 'dbrepo-search-service'"
- "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-search-service-init'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-storage-service'"
- "IGNORE_VOLUMES=1 IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-storage-service-init'"
- "IGNORE_IMAGE=1 bash .scripts/check-service.sh 'dbrepo-ui'"
- "IGNORE_VOLUMES=1 bash .scripts/check-service.sh 'dbrepo-upload-service'"
lint-helm-chart:
image: docker.io/alpine:${ALPINE_VERSION}
stage: lint
before_script:
- apk --no-cache add helm git bash
- cp ./helm/dbrepo/values.schema.json ./CI_values.schema.json
- helm plugin install https://github.com/losisin/helm-values-schema-json.git
- helm dependency update ./helm/seaweedfs
- helm package ./helm/seaweedfs --destination ./build
- helm dependency update ./helm/dbrepo
script:
- bash .scripts/check-helm.sh
- helm lint ./helm/dbrepo
- helm schema -input ./helm/dbrepo/values.yaml -output ./helm/dbrepo/values.schema.json
- diff ./CI_values.schema.json ./helm/dbrepo/values.schema.json
- diff ./dbrepo-metadata-db/1_setup-schema.sql ./helm/dbrepo/files/01-setup-schema.sql
- diff ./dbrepo-auth-service/listeners/target/create-event-listener.jar ./helm/dbrepo/files/create-event-listener.jar
artifacts:
when: always
paths:
- ./helm/dbrepo/values.schema.json
expire_in: 1 days
lint-helm-readme:
image: docker.io/node:${NODE_VERSION}-alpine${ALPINE_VERSION}
stage: lint
before_script:
- apk --no-cache add alpine-sdk bash git
- cp ./helm/dbrepo/README.md ./CI_README.md
- git clone https://github.com/bitnami/readme-generator-for-helm
- (cd ./readme-generator-for-helm && npm install && npm install -g pkg && pkg . -o /usr/local/sbin/readme-generator)
script:
- readme-generator --readme ./helm/dbrepo/README.md --values ./helm/dbrepo/values.yaml
- diff ./CI_README.md ./helm/dbrepo/README.md
artifacts:
when: always
paths:
- ./helm/dbrepo/README.md
expire_in: 1 days
lint-open-api-version:
image: docker.io/alpine:${ALPINE_VERSION}
stage: lint
variables:
VERSION: 4.45.1
BINARY: yq_linux_amd64
before_script:
- 'apk --no-cache add bash wget'
- 'wget https://github.com/mikefarah/yq/releases/download/v${VERSION}/${BINARY} -O /usr/bin/yq && chmod +x /usr/bin/yq'
script:
- yq '.externalDocs.url' ./.docs/.openapi/api.base.yaml | grep "${DOC_VERSION}"
- yq '.info.version' ./.docs/.openapi/api.base.yaml | grep "${DOC_VERSION}"
>>>>>>> Stashed changes
build-metadata-service:
image: maven:3-openjdk-17
stage: build
......
{
"_meta": {
"hash": {
<<<<<<< Updated upstream
"sha256": "9e1fb16b1632a76b8a2fb6ac372b92556c573a7246bd37dd32813559bb27c8d9"
=======
"sha256": "9cc4c161729b642069bbf4ab379c0f4a9122035afcb3ac7b5b1bfc13281f76aa"
>>>>>>> Stashed changes
},
"pipfile-spec": 6,
"requires": {
......@@ -440,7 +444,11 @@
},
"dbrepo": {
"hashes": [
<<<<<<< Updated upstream
"sha256:84607677b0826bb9b2fa120aacdf56d16c8d9ae423f435b2bd2c22b1c965a33c"
=======
"sha256:7f98329f08d1da6fe45da8130cfb3ebd6e947a4101d46f92b31d7204b29a153d"
>>>>>>> Stashed changes
],
"markers": "python_version >= '3.11'",
"path": "./lib/dbrepo-1.4.7.tar.gz"
......
File added
File added
File added
......@@ -6,22 +6,26 @@ LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
FROM maven:3-openjdk-17 AS build
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
<<<<<<< Updated upstream
COPY ./pom.xml ./
RUN mvn -fn -B dependency:go-offline
=======
>>>>>>> Stashed changes
COPY --from=dependency /root/.m2/repository/at/tuwien /root/.m2/repository/at/tuwien
COPY ./querystore ./querystore
COPY ./report ./report
COPY ./rest-service ./rest-service
COPY ./services ./services
COPY ./pom.xml ./
# Make sure it compiles
RUN mvn clean package -DskipTests
###### THIRD STAGE ######
FROM amazoncorretto:17-alpine3.19 AS runtime
FROM amazoncorretto:17-alpine3.21 AS runtime
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
RUN apk add --no-cache curl bash jq
......@@ -30,7 +34,7 @@ WORKDIR /app
USER 1001
COPY --from=build --chown=1001 ./rest-service/target/rest-service-*.jar ./data-service.jar
COPY --from=build --chown=1001 ./rest-service/target/data-service.jar ./data-service.jar
# non-root port
EXPOSE 8080
......
......@@ -249,6 +249,7 @@
</dependencies>
<build>
<finalName>data-service</finalName>
<resources>
<resource>
<directory>${basedir}/src/main/resources</directory>
......
###### FIRST STAGE ######
<<<<<<< Updated upstream
FROM maven:3-openjdk-17 AS build
=======
FROM maven:3-amazoncorretto-17-alpine AS build
>>>>>>> Stashed changes
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
COPY ./pom.xml ./
......@@ -12,7 +16,11 @@ COPY ./rest-service/pom.xml ./rest-service/
COPY ./services/pom.xml ./services/
COPY ./test/pom.xml ./test/
<<<<<<< Updated upstream
RUN mvn verify -B -fn
=======
RUN mvn dependency:go-offline
>>>>>>> Stashed changes
COPY ./api ./api
COPY ./entities ./entities
......@@ -27,7 +35,7 @@ COPY ./test ./test
RUN mvn clean install -DskipTests
###### SECOND STAGE ######
FROM amazoncorretto:17-alpine3.19 AS runtime
FROM amazoncorretto:17-alpine3.21 AS runtime
LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
RUN apk add --no-cache curl bash jq
......@@ -36,9 +44,9 @@ WORKDIR /app
USER 1001
COPY --from=build --chown=1001 ./rest-service/target/dbrepo-metadata-service-rest-service-*.jar ./metadata-service.jar
COPY --from=build --chown=1001 ./rest-service/target/metadata-service.jar ./metadata-service.jar
# non-root port
EXPOSE 8080
ENTRYPOINT ["java", "-Dlog4j2.formatMsgNoLookups=true", "-jar", "./metadata-service.jar"]
ENTRYPOINT ["java", "-jar", "./metadata-service.jar"]
......@@ -14,6 +14,10 @@ import java.time.Instant;
@Getter
@Setter
@Builder
<<<<<<< Updated upstream
=======
@EqualsAndHashCode(callSuper = false)
>>>>>>> Stashed changes
@NoArgsConstructor
@AllArgsConstructor
@Jacksonized
......
......@@ -18,6 +18,10 @@ import java.util.List;
@Getter
@Setter
@Builder
<<<<<<< Updated upstream
=======
@EqualsAndHashCode(callSuper = false)
>>>>>>> Stashed changes
@NoArgsConstructor
@AllArgsConstructor
@Jacksonized
......@@ -59,6 +63,14 @@ public class DatabaseDto {
@ToString.Exclude
@NotNull
<<<<<<< Updated upstream
=======
@JsonProperty("is_schema_public")
@Schema(example = "true")
private Boolean isSchemaPublic;
@NotNull
>>>>>>> Stashed changes
private ContainerDto container;
private List<DatabaseAccessDto> accesses;
......
......@@ -19,6 +19,10 @@ import java.util.UUID;
@Getter
@Setter
@Builder
<<<<<<< Updated upstream
=======
@EqualsAndHashCode(callSuper = false)
>>>>>>> Stashed changes
@NoArgsConstructor
@AllArgsConstructor
@Jacksonized
......@@ -64,6 +68,7 @@ public class ViewDto {
@Schema(example = "7de03e818900b6ea6d58ad0306d4a741d658c6df3d1964e89ed2395d8c7e7916")
private String queryHash;
<<<<<<< Updated upstream
@NotNull
@Schema(example = "2021-03-12T15:26:21Z")
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC")
......@@ -71,6 +76,10 @@ public class ViewDto {
@JsonIgnore
private UUID createdBy;
=======
@ToString.Exclude
private DatabaseDto database;
>>>>>>> Stashed changes
@NotNull
private UserDto creator;
......
......@@ -21,6 +21,10 @@ import java.util.UUID;
@Getter
@Setter
@Builder
<<<<<<< Updated upstream
=======
@EqualsAndHashCode(callSuper = false)
>>>>>>> Stashed changes
@NoArgsConstructor
@AllArgsConstructor
@Jacksonized
......@@ -110,8 +114,13 @@ public class TableDto {
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC")
private Instant created;
<<<<<<< Updated upstream
@NotNull
private List<ColumnDto> columns;
=======
@ToString.Exclude
private DatabaseDto database;
>>>>>>> Stashed changes
@NotNull
private ConstraintsDto constraints;
......
......@@ -12,6 +12,10 @@ import java.util.UUID;
@Getter
@Setter
@Builder
<<<<<<< Updated upstream
=======
@EqualsAndHashCode(callSuper = false)
>>>>>>> Stashed changes
@NoArgsConstructor
@AllArgsConstructor
@Jacksonized
......
......@@ -61,7 +61,10 @@ public class Container {
@Column
private String uiAdditionalFlags;
<<<<<<< Updated upstream
@ToString.Exclude
=======
>>>>>>> Stashed changes
@OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.MERGE)
@JoinColumns({
@JoinColumn(name = "cid", referencedColumnName = "id", insertable = false, updatable = false)
......
......@@ -23,7 +23,7 @@ import java.util.UUID;
@Data
@Entity
@Builder
@Builder(toBuilder = true)
@Log4j2
@ToString
@AllArgsConstructor
......
......@@ -119,6 +119,14 @@ public class TableColumn implements Comparable<TableColumn> {
@Column(name = "std_dev")
private BigDecimal stdDev;
<<<<<<< Updated upstream
=======
@CreatedDate
@Column(nullable = false, updatable = false, columnDefinition = "TIMESTAMP default NOW()")
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC")
private Instant created;
>>>>>>> Stashed changes
@LastModifiedDate
@Column(columnDefinition = "TIMESTAMP")
private Instant lastModified;
......
......@@ -96,6 +96,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-webflux</artifactId>
</dependency>
<!-- Open API -->
<dependency>
<groupId>org.springdoc</groupId>
......@@ -268,6 +272,7 @@
</dependencies>
<build>
<finalName>metadata-service</finalName>
<resources>
<resource>
<directory>${basedir}/src/main/resources</directory>
......
......@@ -4,7 +4,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseStatus(code = HttpStatus.BAD_GATEWAY, reason = "error.search.connection")
public class SearchServiceConnectionException extends Exception {
public class SearchServiceConnectionException extends RuntimeException {
public SearchServiceConnectionException(String msg) {
super(msg);
......
......@@ -4,7 +4,7 @@ import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@ResponseStatus(code = HttpStatus.SERVICE_UNAVAILABLE, reason = "error.search.invalid")
public class SearchServiceException extends Exception {
public class SearchServiceException extends RuntimeException {
public SearchServiceException(String message) {
super(message);
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment