diff --git a/dbrepo-auth-service/listeners/create-event-listener.jar b/dbrepo-auth-service/listeners/create-event-listener.jar index fb1458371a5ce51be8ca63a9421a621e0ef09ce3..b3d777647147955c09ad147041a6f513aa5a5d4f 100644 Binary files a/dbrepo-auth-service/listeners/create-event-listener.jar and b/dbrepo-auth-service/listeners/create-event-listener.jar differ diff --git a/dbrepo-data-service/Dockerfile b/dbrepo-data-service/Dockerfile index 32cdfc7d7a8b0f5b463012b22fe00591d7018f7f..3846f268aa28c662f26d3514e62cdf2af85481e7 100644 --- a/dbrepo-data-service/Dockerfile +++ b/dbrepo-data-service/Dockerfile @@ -27,9 +27,8 @@ RUN apk --no-cache add \ WORKDIR /app RUN mkdir -p /var/log/app/service/data && \ - chown -R 1001:1001 /var/log/app/service/data - -RUN adduser -D dbrepo --uid 1001 && \ + chown -R 1001:1001 /var/log/app/service/data && \ + adduser -D dbrepo --uid 1001 && \ chown -R 1001:1001 /app USER 1001 diff --git a/dbrepo-data-service/pom.xml b/dbrepo-data-service/pom.xml index 8175e571ab49f8df326510662bfdad3b01789bfd..05acab4edb70ab1e12fe5dd80ef6935af43c9d63 100644 --- a/dbrepo-data-service/pom.xml +++ b/dbrepo-data-service/pom.xml @@ -58,7 +58,7 @@ <aws-s3.version>2.25.23</aws-s3.version> <minio.version>8.5.7</minio.version> <guava.version>33.3.0-jre</guava.version> - <spark.version>4.0.0-preview2</spark.version> + <spark.version>4.0.0</spark.version> <keycloak.version>26.0.4</keycloak.version> <scala.version>2.13</scala.version> <antlr-runtime.version>3.5.2</antlr-runtime.version> diff --git a/dbrepo-data-service/rest-service/src/main/resources/application.yml b/dbrepo-data-service/rest-service/src/main/resources/application.yml index be28c4ba71d4375450609998ee63a3f301b611dd..d0ed0105358e4e489a5dcf65797c4b5313059745 100644 --- a/dbrepo-data-service/rest-service/src/main/resources/application.yml +++ b/dbrepo-data-service/rest-service/src/main/resources/application.yml @@ -65,7 +65,7 @@ dbrepo: analyseService: "${ANALYSE_SERVICE_ENDPOINT:http://analyse-service:8080}" storageService: "${S3_ENDPOINT:http://storage-service:9000}" authService: "${AUTH_SERVICE_ENDPOINT:http://auth-service:8080}" - computeService: "${COMPUTE_SERVICE_ENDPOINT:local[*]}" + computeService: "${COMPUTE_SERVICE_ENDPOINT:spark://compute-service:7077}" s3: accessKeyId: "${S3_ACCESS_KEY_ID:seaweedfsadmin}" secretAccessKey: "${S3_SECRET_ACCESS_KEY:seaweedfsadmin}" diff --git a/dbrepo-data-service/services/src/main/java/at/ac/tuwien/ifs/dbrepo/config/SparkConfig.java b/dbrepo-data-service/services/src/main/java/at/ac/tuwien/ifs/dbrepo/config/SparkConfig.java index 12640fbc42c10211beecacde41708ce5fa718cbd..f058c985b0a2b97ef955300b2c2cf18137ca84c6 100644 --- a/dbrepo-data-service/services/src/main/java/at/ac/tuwien/ifs/dbrepo/config/SparkConfig.java +++ b/dbrepo-data-service/services/src/main/java/at/ac/tuwien/ifs/dbrepo/config/SparkConfig.java @@ -50,7 +50,7 @@ public class SparkConfig { public SparkConf sparkConf() { final SparkConf config = new SparkConf() .setMaster(computeEndpoint) - .setAppName("dbrepo-data-service"); + .setAppName("data-service"); sparkOptions() .forEach(config::set); return config; diff --git a/docker-compose.yml b/docker-compose.yml index aec93aa4fd17a91845b6ca0be5138ff2a090559b..4ffc862843b6c447ce94f86d1991da9c4bc8795b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -565,7 +565,7 @@ services: BROKER_USERNAME: "${SYSTEM_USERNAME:-admin}" BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}" CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000} - COMPUTE_SERVICE_ENDPOINT: "${COMPUTE_SERVICE_ENDPOINT:-local[2]}" + COMPUTE_SERVICE_ENDPOINT: "${COMPUTE_SERVICE_ENDPOINT:-spark://compute-service:7077}" EXCHANGE_NAME: "${EXCHANGE_NAME:-dbrepo}" METADATA_SERVICE_ENDPOINT: "${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}" GRANT_DEFAULT_READ: "${GRANT_DEFAULT_READ:-SELECT, EXECUTE}" @@ -648,3 +648,29 @@ services: condition: service_healthy logging: driver: json-file + + dbrepo-compute-service: + restart: "no" + container_name: dbrepo-compute-service + hostname: compute-service + image: docker.io/bitnami/spark:4.0.0 + ports: + - 8088:8080 + environment: + SPARK_MODE: "master" + logging: + driver: json-file + + dbrepo-compute-service-worker: + restart: "no" + container_name: dbrepo-compute-service-worker + hostname: compute-service-worker + image: docker.io/bitnami/spark:4.0.0 + environment: + SPARK_MODE: "worker" + SPARK_MASTER_URL: "spark://compute-service:7077" + depends_on: + dbrepo-compute-service: + condition: service_started + logging: + driver: json-file diff --git a/docs/stylesheets/.sass-cache/10990fa183107f4149f38216a4d00fe324a8131e/extra.scssc b/docs/stylesheets/.sass-cache/10990fa183107f4149f38216a4d00fe324a8131e/extra.scssc new file mode 100644 index 0000000000000000000000000000000000000000..470bc88b85d636ed100a0939406a9e536a89b6c8 Binary files /dev/null and b/docs/stylesheets/.sass-cache/10990fa183107f4149f38216a4d00fe324a8131e/extra.scssc differ diff --git a/helm/dbrepo/files/create-event-listener.jar b/helm/dbrepo/files/create-event-listener.jar index fb1458371a5ce51be8ca63a9421a621e0ef09ce3..b3d777647147955c09ad147041a6f513aa5a5d4f 100644 Binary files a/helm/dbrepo/files/create-event-listener.jar and b/helm/dbrepo/files/create-event-listener.jar differ