WIP

Signed-off-by: Martin Weise <martin.weise@tuwien.ac.at>

dbrepo-data-service/Dockerfile

@@ -27,9 +27,8 @@ RUN apk --no-cache add \
 WORKDIR /app
 
 RUN mkdir -p /var/log/app/service/data && \
-    chown -R 1001:1001 /var/log/app/service/data
-
-RUN adduser -D dbrepo --uid 1001 && \
+    chown -R 1001:1001 /var/log/app/service/data && \
+    adduser -D dbrepo --uid 1001 && \
     chown -R 1001:1001 /app
 
 USER 1001

dbrepo-data-service/pom.xml

@@ -58,7 +58,7 @@
         <aws-s3.version>2.25.23</aws-s3.version>
         <minio.version>8.5.7</minio.version>
         <guava.version>33.3.0-jre</guava.version>
-        <spark.version>4.0.0-preview2</spark.version>
+        <spark.version>4.0.0</spark.version>
         <keycloak.version>26.0.4</keycloak.version>
         <scala.version>2.13</scala.version>
         <antlr-runtime.version>3.5.2</antlr-runtime.version>

dbrepo-data-service/rest-service/src/main/resources/application.yml

@@ -65,7 +65,7 @@ dbrepo:
     analyseService: "${ANALYSE_SERVICE_ENDPOINT:http://analyse-service:8080}"
     storageService: "${S3_ENDPOINT:http://storage-service:9000}"
     authService: "${AUTH_SERVICE_ENDPOINT:http://auth-service:8080}"
-    computeService: "${COMPUTE_SERVICE_ENDPOINT:local[*]}"
+    computeService: "${COMPUTE_SERVICE_ENDPOINT:spark://compute-service:7077}"
   s3:
     accessKeyId: "${S3_ACCESS_KEY_ID:seaweedfsadmin}"
     secretAccessKey: "${S3_SECRET_ACCESS_KEY:seaweedfsadmin}"

dbrepo-data-service/services/src/main/java/at/ac/tuwien/ifs/dbrepo/config/SparkConfig.java

@@ -50,7 +50,7 @@ public class SparkConfig {
     public SparkConf sparkConf() {
         final SparkConf config = new SparkConf()
                 .setMaster(computeEndpoint)
-                .setAppName("dbrepo-data-service");
+                .setAppName("data-service");
         sparkOptions()
                 .forEach(config::set);
         return config;

docker-compose.yml

@@ -565,7 +565,7 @@ services:
       BROKER_USERNAME: "${SYSTEM_USERNAME:-admin}"
       BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}"
       CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000}
-      COMPUTE_SERVICE_ENDPOINT: "${COMPUTE_SERVICE_ENDPOINT:-local[2]}"
+      COMPUTE_SERVICE_ENDPOINT: "${COMPUTE_SERVICE_ENDPOINT:-spark://compute-service:7077}"
       EXCHANGE_NAME: "${EXCHANGE_NAME:-dbrepo}"
       METADATA_SERVICE_ENDPOINT: "${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}"
       GRANT_DEFAULT_READ: "${GRANT_DEFAULT_READ:-SELECT, EXECUTE}"
@@ -648,3 +648,29 @@ services:
         condition: service_healthy
     logging:
       driver: json-file
+
+  dbrepo-compute-service:
+    restart: "no"
+    container_name: dbrepo-compute-service
+    hostname: compute-service
+    image: docker.io/bitnami/spark:4.0.0
+    ports:
+      - 8088:8080
+    environment:
+      SPARK_MODE: "master"
+    logging:
+      driver: json-file
+
+  dbrepo-compute-service-worker:
+    restart: "no"
+    container_name: dbrepo-compute-service-worker
+    hostname: compute-service-worker
+    image: docker.io/bitnami/spark:4.0.0
+    environment:
+      SPARK_MODE: "worker"
+      SPARK_MASTER_URL: "spark://compute-service:7077"
+    depends_on:
+      dbrepo-compute-service:
+        condition: service_started
+    logging:
+      driver: json-file