Merge branch 'master' into release-1.4.5

.docker/docker-compose.yml

@@ -104,19 +104,19 @@ services:
       - "${SHARED_VOLUME:-/tmp}:/tmp"
     environment:
       ADMIN_EMAIL: "${ADMIN_EMAIL:-noreply@localhost}"
-      ANALYSE_SERVICE_ENDPOINT: "${ANALYSE_SERVICE_ENDPOINT:-http://gateway-service}"
+      ANALYSE_SERVICE_ENDPOINT: "${ANALYSE_SERVICE_ENDPOINT:-http://analyse-service:8080}"
       AUTH_SERVICE_ADMIN: ${AUTH_SERVICE_ADMIN:-admin}
       AUTH_SERVICE_ADMIN_PASSWORD: ${AUTH_SERVICE_ADMIN_PASSWORD:-admin}
       AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
       AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT_SECRET:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
-      AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://gateway-service/api/auth}
+      AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
       BASE_URL: "${BASE_URL:-http://localhost}"
       BROKER_EXCHANGE_NAME: ${BROKER_EXCHANGE_NAME:-dbrepo}
       BROKER_QUEUE_NAME: ${BROKER_QUEUE_NAME:-dbrepo}
       BROKER_HOST: "${BROKER_ENDPOINT:-broker-service}"
       BROKER_PASSWORD: ${BROKER_PASSWORD:-admin}
       BROKER_PORT: ${BROKER_PORT:-5672}
-      BROKER_SERVICE_ENDPOINT: ${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}
+      BROKER_SERVICE_ENDPOINT: ${BROKER_SERVICE_ENDPOINT:-http://broker-service:15672}
       BROKER_USERNAME: ${BROKER_USERNAME:-admin}
       BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}"
       CROSSREF_ENDPOINT: "${CROSSREF_ENDPOINT:-http://data.crossref.org}"
@@ -126,14 +126,14 @@ services:
       JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"
       LOG_LEVEL: ${LOG_LEVEL:-info}
       METADATA_DB: "${METADATA_DB:-dbrepo}"
+      METADATA_DB_PASSWORD: "${METADATA_DB_PASSWORD:-dbrepo}"
       METADATA_HOST: "${METADATA_HOST:-metadata-db}"
       METADATA_JDBC_EXTRA_ARGS: "${METADATA_JDBC_EXTRA_ARGS:-}"
+      METADATA_PORT: "${METADATA_PORT:-3306}"
       METADATA_USERNAME: root
-      METADATA_DB_PASSWORD: "${METADATA_DB_PASSWORD:-dbrepo}"
-      PID_BASE: ${PID_BASE:-http://localhost/pid/}
       REPOSITORY_NAME: "${REPOSITORY_NAME:-Database Repository}"
       ROR_ENDPOINT: "${ROR_ENDPOINT:-https://api.ror.org}"
-      SEARCH_SERVICE_ENDPOINT: "${SEARCH_SERVICE_ENDPOINT:-http://gateway-service}"
+      SEARCH_SERVICE_ENDPOINT: "${SEARCH_SERVICE_ENDPOINT:-http://search-service:8080}"
       S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
       S3_BUCKET: "${S3_BUCKET:-dbrepo}"
       S3_ENDPOINT: "${S3_ENDPOINT:-http://storage-service:9000}"
@@ -167,7 +167,6 @@ services:
       AUTH_SERVICE_CLIENT: ${AUTH_SERVICE_CLIENT:-dbrepo-client}
       AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
       AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
-      GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
       JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"
       S3_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID:-seaweedfsadmin}"
       S3_BUCKET: "${S3_BUCKET:-dbrepo}"
@@ -243,7 +242,7 @@ services:
       AUTH_SERVICE_CLIENT_SECRET: ${AUTH_SERVICE_CLIENT_SECRET:-MUwRc7yfXSJwX8AdRMWaQC3Nep1VjwgG}
       AUTH_SERVICE_ENDPOINT: ${AUTH_SERVICE_ENDPOINT:-http://auth-service:8080}
       COLLECTION: ${COLLECTION:-['database','table','column','identifier','unit','concept','user','view']}
-      GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
+      METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
       OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
       OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200}
       OPENSEARCH_USERNAME: ${SEARCH_DB_USERNAME:-admin}
@@ -342,12 +341,11 @@ services:
     hostname: search-service-init
     image: registry.datalab.tuwien.ac.at/dbrepo/search-service-init:1.4.5
     environment:
-      GATEWAY_SERVICE_ENDPOINT: ${GATEWAY_SERVICE_ENDPOINT:-http://gateway-service}
+      METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
       OPENSEARCH_HOST: ${OPENSEARCH_HOST:-search-db}
       OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200}
       OPENSEARCH_USERNAME: ${SEARCH_DB_USERNAME:-admin}
       OPENSEARCH_PASSWORD: ${SEARCH_DB_PASSWORD:-admin}
-      LOG_LEVEL: ${LOG_LEVEL:-info}
     depends_on:
       dbrepo-search-db:
         condition: service_healthy
@@ -429,14 +427,14 @@ services:
       BROKER_EXCHANGE_NAME: ${BROKER_EXCHANGE_NAME:-dbrepo}
       BROKER_QUEUE_NAME: ${BROKER_QUEUE_NAME:-dbrepo}
       BROKER_HOST: "${BROKER_ENDPOINT:-broker-service}"
-      BROKER_PASSWORD: ${SYSTEM_USERNAME:-admin}
+      BROKER_PASSWORD: ${SYSTEM_PASSWORD:-admin}
       BROKER_PORT: ${BROKER_PORT:-5672}
       BROKER_SERVICE_ENDPOINT: ${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}
-      BROKER_USERNAME: ${SYSTEM_PASSWORD:-admin}
+      BROKER_USERNAME: ${SYSTEM_USERNAME:-admin}
       BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}"
       CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000}
       EXCHANGE_NAME: ${EXCHANGE_NAME:-dbrepo}
-      METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://gateway-service}
+      METADATA_SERVICE_ENDPOINT: ${METADATA_SERVICE_ENDPOINT:-http://metadata-service:8080}
       GRANT_DEFAULT_READ: "${GRANT_DEFAULT_READ:-SELECT}"
       GRANT_DEFAULT_WRITE: "${GRANT_DEFAULT_WRITE:-SELECT, CREATE, CREATE VIEW, CREATE ROUTINE, CREATE TEMPORARY TABLES, LOCK TABLES, INDEX, TRIGGER, INSERT, UPDATE, DELETE}"
       JWT_PUBKEY: "${JWT_PUBKEY:-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqqnHQ2BWWW9vDNLRCcxD++xZg/16oqMo/c1l+lcFEjjAIJjJp/HqrPYU/U9GvquGE6PbVFtTzW1KcKawOW+FJNOA3CGo8Q1TFEfz43B8rZpKsFbJKvQGVv1Z4HaKPvLUm7iMm8Hv91cLduuoWx6Q3DPe2vg13GKKEZe7UFghF+0T9u8EKzA/XqQ0OiICmsmYPbwvf9N3bCKsB/Y10EYmZRb8IhCoV9mmO5TxgWgiuNeCTtNCv2ePYqL/U0WvyGFW0reasIK8eg3KrAUj8DpyOgPOVBn3lBGf+3KFSYi+0bwZbJZWqbC/Xlk20Go1YfeJPRIt7ImxD27R/lNjgDO/MwIDAQAB}"

.docs/api/auth-service.md

@@ -19,10 +19,37 @@ of immutable properties (id, username) is mirrored in the [Metadata Database](..
 
 ## Identities
 
-:octicons-tag-16:{ title="Minimum version" } 1.4.4
-
-Identities can also be added in Keycloak directly. When requesting a JWT token from the `/api/user` endpoint, the
-immutable properties mentioned in c.f. [Overview](#overview) are copied transparent to the user on first login.
+:octicons-tag-16:{ title="Minimum version" } 1.4.5
+
+Identities are managed via LDAP through the [Identity Service](../identity-service). The normal workflow is that the
+[Metadata Service](../metadata-service) adds identities when user register. In some cases, where this is not possible
+(e.g. in workshop-scenarios where accounts are created before the workshop starts), identities need to be created
+manually in Keycloak. The recommended workflow is:
+
+1. Login to the Auth Service as **Admin** and in the dbrepo realm navigate to **Users**
+2. Click the **Add user** button and fill out the Username field and assign the group `researchers` by clicking 
+   the **Join Groups** and selecting it. Click **Join** and **Create**.
+3. Click the **Credentials** tab above and **Set password**. In the popup window assign a secure password to the user
+   and set **Temporary** to `Off`.
+
+    !!! example "Create user with specific id"
+
+        The user id is created automatically. In case you need to create a user with specific id such as in migration
+        scenarios, you need to change the `entryUUID` in the [Identity Service](../identity-service) by modifying this
+        protected attribute in `relax` mode:
+
+        ```bash
+        echo "dn: uid=<username>,ou=users,dc=dbrepo,dc=at
+        changetype: modify
+        replace: entryUUID
+        entryUUID: 506ae590-11a2-4d2d-82b8-45121c6b4dab" | \
+        ldapmodify -h localhost -p 1389 -D cn=admin,dc=dbrepo,dc=at -c -x -e relax \
+        -w<adminpassword> 
+        ```
+
+4. Finally you need to query the user info once by navigating again to **Users**
+   and search for the **Username** and click :arrow_right: to search. Click the username and ensure that the 
+   **User metadata** contains the entry **LDAP_ID**.
 
 ## Groups
 

dbrepo-broker-service/rabbitmq.conf

 # user
 default_vhost = dbrepo
 default_user_tags.administrator = false
-default_permissions.configure = .*
-default_permissions.read = .*
-default_permissions.write = .*
 
 # enable http outside localhost
 listeners.tcp.1 = 0.0.0.0:5672
@@ -15,6 +12,7 @@ management.load_definitions = /app/definitions.json
 # logging
 log.console = true
 log.console.level = warning
+auth_ldap.log = true
 
 # Obviously your authentication server cannot vouch for itself, so you'll need another backend with at least one user in
 # it. You should probably use the internal database
@@ -26,7 +24,7 @@ auth_backends.2 = internal
 auth_ldap.servers.1 = identity-service
 auth_ldap.port = 1389
 auth_ldap.user_dn_pattern = ${username}
-auth_ldap.dn_lookup_base = ou=users,dc=dbrepo,dc=at
+auth_ldap.dn_lookup_base = dc=dbrepo,dc=at
 auth_ldap.dn_lookup_attribute = uid
 auth_ldap.dn_lookup_bind.user_dn = cn=admin,dc=dbrepo,dc=at
 auth_ldap.dn_lookup_bind.password = admin

dbrepo-metadata-service/rest-service/src/main/java/at/tuwien/endpoints/UserEndpoint.java

@@ -326,7 +326,7 @@ public class UserEndpoint {
     }
 
     @PutMapping("/{userId}/password")
-    @Transactional
+    @Transactional(rollbackFor = {Exception.class})
     @PreAuthorize("isAuthenticated()")
     @Observed(name = "dbrepo_user_password_modify")
     @Operation(summary = "Update user password",
@@ -367,17 +367,16 @@ public class UserEndpoint {
             AuthServiceConnectionException, UserNotFoundException, DatabaseNotFoundException, DataServiceException,
             DataServiceConnectionException, CredentialsInvalidException {
         log.debug("endpoint modify a user password, userId={}, data.password=(hidden)", userId);
-        User user = userService.findById(userId);
+        final User user = userService.findById(userId);
         if (!user.equals(principal)) {
             log.error("Failed to modify user password: not current user");
             throw new NotAllowedException("Failed to modify user password: not current user");
         }
-        user = userService.findByUsername(principal.getName());
-        userService.updatePassword(user, data);
         authenticationService.updatePassword(user, data);
         for (Database database : databaseService.findAllAccess(userId)) {
             databaseService.updatePassword(database, user);
         }
+        userService.updatePassword(user, data);
         return ResponseEntity.accepted()
                 .build();
     }

dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/gateway/KeycloakGatewayUnitTest.java

@@ -191,7 +191,8 @@ public class KeycloakGatewayUnitTest extends AbstractUnitTest {
     }
 
     @Test
-    public void updateUserCredentials_succeeds() throws AuthServiceException, AuthServiceConnectionException {
+    public void updateUserCredentials_succeeds() throws AuthServiceException, AuthServiceConnectionException,
+            UserNotFoundException {
 
         /* mock */
         when(restTemplate.exchange(anyString(), eq(HttpMethod.POST), any(HttpEntity.class), eq(TokenDto.class)))

dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/AuthenticationServiceIntegrationTest.java

@@ -57,6 +57,7 @@ public class AuthenticationServiceIntegrationTest extends AbstractUnitTest {
         keycloakGateway.createUser(USER_1_KEYCLOAK_SIGNUP_REQUEST);
         final User request = User.builder()
                 .id(keycloakGateway.findByUsername(USER_1_USERNAME).getId())
+                .username(USER_1_USERNAME)
                 .build();
 
         /* test */

dbrepo-metadata-service/rest-service/src/test/java/at/tuwien/service/UserServiceUnitTest.java

@@ -109,7 +109,7 @@ public class UserServiceUnitTest extends AbstractUnitTest {
 
     @Test
     public void updatePassword_succeeds() throws AuthServiceException, AuthServiceConnectionException,
-            CredentialsInvalidException {
+            UserNotFoundException {
 
         /* mock */
         doNothing()

dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/KeycloakGateway.java

@@ -40,7 +40,7 @@ public interface KeycloakGateway {
      * @param password The user credential.
      */
     void updateUserCredentials(UUID id, UserPasswordDto password) throws AuthServiceException,
-            AuthServiceConnectionException;
+            AuthServiceConnectionException, UserNotFoundException;
 
     /**
      * Finds a user in the metadata database by given username.

dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java

@@ -161,7 +161,7 @@ public class KeycloakGatewayImpl implements KeycloakGateway {
 
     @Override
     public void updateUserCredentials(UUID id, UserPasswordDto data) throws AuthServiceException,
-            AuthServiceConnectionException {
+            AuthServiceConnectionException, UserNotFoundException {
         final UpdateCredentialsDto payload = metadataMapper.passwordToUpdateCredentialsDto(data.getPassword());
         final String path = "/admin/realms/dbrepo/users/" + id;
         log.trace("update user credentials at endpoint {} with path {}", keycloakConfig.getKeycloakEndpoint(), path);
@@ -171,6 +171,9 @@ public class KeycloakGatewayImpl implements KeycloakGateway {
         } catch (HttpServerErrorException e) {
             log.error("Failed to update user credentials: {}", e.getMessage());
             throw new AuthServiceConnectionException("Service unavailable", e);
+        } catch (HttpClientErrorException.NotFound e) {
+            log.error("Failed to update user credentials: user not found: {}", e.getMessage());
+            throw new UserNotFoundException("User not found", e);
         } catch (Exception e) {
             log.error("Failed to update user: unexpected response: {}", e.getMessage());
             throw new AuthServiceException("Unexpected result", e);

dbrepo-metadata-service/services/src/main/java/at/tuwien/service/AuthenticationService.java

@@ -60,5 +60,6 @@ public interface AuthenticationService {
      * @throws AuthServiceException           The auth service responded with unexpected behavior.
      * @throws AuthServiceConnectionException The connection with the auth service could not be established.
      */
-    void updatePassword(User user, UserPasswordDto data) throws AuthServiceException, AuthServiceConnectionException, CredentialsInvalidException;
+    void updatePassword(User user, UserPasswordDto data) throws AuthServiceException, AuthServiceConnectionException,
+            CredentialsInvalidException, UserNotFoundException;
 }

dbrepo-metadata-service/services/src/main/java/at/tuwien/service/impl/AuthenticationServiceImpl.java

@@ -43,7 +43,8 @@ public class AuthenticationServiceImpl implements AuthenticationService {
     @Override
     public void delete(User user) throws AuthServiceException, AuthServiceConnectionException, UserNotFoundException,
             CredentialsInvalidException {
-        keycloakGateway.deleteUser(user.getId());
+        final UserDto keycloakUser = findByUsername(user.getUsername());
+        keycloakGateway.deleteUser(keycloakUser.getId());
     }
 
     @Override
@@ -72,8 +73,9 @@ public class AuthenticationServiceImpl implements AuthenticationService {
 
     @Override
     public void updatePassword(User user, UserPasswordDto data) throws AuthServiceException,
-            AuthServiceConnectionException, CredentialsInvalidException {
-        keycloakGateway.updateUserCredentials(user.getId(), data);
+            AuthServiceConnectionException, CredentialsInvalidException, UserNotFoundException {
+        final UserDto keycloakUser = findByUsername(user.getUsername());
+        keycloakGateway.updateUserCredentials(keycloakUser.getId(), data);
     }
 
 }

docker-compose.yml

@@ -374,7 +374,6 @@ services:
       - '1389:1389'
       - '1636:1636'
     environment:
-      BITNAMI_DEBUG: true
       LDAP_ADMIN_USERNAME: "${IDENTITY_SERVICE_ADMIN_USERNAME:-admin}"
       LDAP_ADMIN_PASSWORD: "${IDENTITY_SERVICE_ADMIN_PASSWORD:-admin}"
       LDAP_USERS: "${SYSTEM_USERNAME:-admin}"
@@ -404,7 +403,6 @@ services:
       OPENSEARCH_PORT: ${OPENSEARCH_PORT:-9200}
       OPENSEARCH_USERNAME: ${SEARCH_DB_USERNAME:-admin}
       OPENSEARCH_PASSWORD: ${SEARCH_DB_PASSWORD:-admin}
-      LOG_LEVEL: ${LOG_LEVEL:-info}
     depends_on:
       dbrepo-search-db:
         condition: service_healthy
@@ -494,10 +492,10 @@ services:
       BROKER_EXCHANGE_NAME: ${BROKER_EXCHANGE_NAME:-dbrepo}
       BROKER_QUEUE_NAME: ${BROKER_QUEUE_NAME:-dbrepo}
       BROKER_HOST: "${BROKER_ENDPOINT:-broker-service}"
-      BROKER_PASSWORD: ${SYSTEM_USERNAME:-admin}
+      BROKER_PASSWORD: ${SYSTEM_PASSWORD:-admin}
       BROKER_PORT: ${BROKER_PORT:-5672}
       BROKER_SERVICE_ENDPOINT: ${BROKER_SERVICE_ENDPOINT:-http://gateway-service/admin/broker}
-      BROKER_USERNAME: ${SYSTEM_PASSWORD:-admin}
+      BROKER_USERNAME: ${SYSTEM_USERNAME:-admin}
       BROKER_VIRTUALHOST: "${BROKER_VIRTUALHOST:-dbrepo}"
       CONNECTION_TIMEOUT: ${CONNECTION_TIMEOUT:-60000}
       EXCHANGE_NAME: ${EXCHANGE_NAME:-dbrepo}

helm/dbrepo/values.yaml

@@ -306,7 +306,7 @@ brokerservice:
     ## @param brokerservice.ldap.uidField The field containing the user id.
     uidField: uid
     ## @param brokerservice.ldap.basedn The base domain name containing the users.
-    basedn: ou=users,dc=dbrepo,dc=at
+    basedn: dc=dbrepo,dc=at
     ## @param brokerservice.ldap.userDnPattern The pattern to determine the user.
     userDnPattern: ${username}
   auth: