Merge remote-tracking branch 'origin/add-docker-build'

.gitignore

@@ -36,3 +36,4 @@ out/
 ### VS Code ###
 .vscode/
 /dev/gpas*
+/deploy/.env

README.md

@@ -26,7 +26,7 @@ Siehe hierzu auch: https://github.com/CCC-MF/kafka-to-bwhc
 
 ## Pseudonymisierung der Patienten-ID
 
-Wenn eine URI zu einer gPAS-Instanz angegeben ist, wird diese verwendet.
+Wenn eine URI zu einer gPAS-Instanz (Version >= 2023.1.0) angegeben ist, wird diese verwendet.
 Ist diese nicht gesetzt. wird intern eine Anonymisierung der Patienten-ID vorgenommen.
 
 * `APP_PSEUDONYMIZE_PREFIX`: Standortbezogenes Prefix - `UNKNOWN`, wenn nicht gesetzt
@@ -42,7 +42,8 @@ als Patienten-Pseudonym verwendet.
 
 Wurde die Verwendung von gPAS konfiguriert, so sind weitere Angaben zu konfigurieren.
 
-* `APP_PSEUDONYMIZE_GPAS_URI`: URI der gPAS-Instanz inklusive Endpoint (z.B. `http://localhost:8080/ttp-fhir/fhir/gpas/$pseudonymizeAllowCreate`)
+* `APP_PSEUDONYMIZE_GPAS_URI`: URI der gPAS-Instanz inklusive Endpoint (
+  z.B. `http://localhost:8080/ttp-fhir/fhir/gpas/$$pseudonymizeAllowCreate`) 
 * `APP_PSEUDONYMIZE_GPAS_TARGET`: gPas Domänenname
 * `APP_PSEUDONYMIZE_GPAS_USERNAME`: gPas Basic-Auth Benutzername
 * `APP_PSEUDONYMIZE_GPAS_PASSWORD`: gPas Basic-Auth Passwort
@@ -120,6 +121,25 @@ ein Consent-Widerspruch erfolgte.
 
 Diese Anwendung ist auch als Docker-Image verfügbar: https://github.com/CCC-MF/etl-processor/pkgs/container/etl-processor
 
+### Images lokal bauen 
+
+```bash
+./gradlew bootBuildImage
+```
+
+## Deployment
+*Ausführen als Docker Conatiner:*
+
+```bash
+cd ./deploy
+cp env-sample.env .env
+```
+Wenn gewünscht, Änderungen in der `.env` vornehmen.
+
+```bash
+docker compose up -d
+```
+
 ## Entwicklungssetup
 
 Zum Starten einer lokalen Entwicklungs- und Testumgebung kann die beiliegende Datei `dev-compose.yml` verwendet werden.

build.gradle.kts

@@ -60,6 +60,8 @@ dependencies {
     implementation("org.springframework.boot:spring-boot-starter-data-jdbc")
     implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
     implementation("org.springframework.kafka:spring-kafka")
+    // fix CVE-2022-1471
+    implementation("org.yaml:snakeyaml:2.1")
     implementation("org.flywaydb:flyway-mysql")
     implementation("commons-codec:commons-codec")
     implementation("io.projectreactor.kotlin:reactor-kotlin-extensions")

deploy/docker-compose.yaml

@@ -0,0 +1,55 @@
+
+
+services:
+  dnpm-etl-processor:
+    image: ghcr.io/ccc-mf/etl-processor:latest
+    environment:
+      LOGGING_LEVEL_DEV: ${DNPM_LOG_LEVEL:-INFO}
+      SPRING_KAFKA_SECURITY_PROTOCOL: ${DNPM_KAFKA_SECURITY_PROTOCOL:-SSL}
+      SPRING_KAFKA_SSL_TRUST-STORE-TYPE: PKCS12
+      SPRING_KAFKA_SSL_TRUST-STORE-LOCATION: /opt/dnpm-processor/ssl/truststore.jks
+      SPRING_KAFKA_SSL_TRUST-STORE-PASSWORD: ${KAFKA_TRUST_STORE_PASSWORD}
+      SPRING_KAFKA_SSL_KEY-STORE-TYPE: PKCS12
+      SPRING_KAFKA_SSL_KEY-STORE-LOCATION: /opt/dnpm-processor/ssl/keystore.jks
+      SPRING_KAFKA_SSL_KEY-STORE-PASSWORD: ${DNPM_PROCESSOR_KEY_STORE_PASSWORD}
+      SPRING_KAFKA_PRODUCER_COMPRESSION-TYPE: gzip
+      APP_KAFKA_TOPIC: ${DNPM_KAFKA_TOPIC}
+      APP_KAFKA_SERVERS: ${KAFKA_BROKERS}
+      APP_KAFKA_GROUP_ID: ${DNPM_KAFKA_GROUP_ID}
+      APP_KAFKA_RESPONSE_TOPIC: ${DNPM_KAFKA_RESPONSE_TOPIC}
+      APP_REST_URI: ${DNPM_BWHC_REST_URI}
+      SPRING_DATASOURCE_URL: ${DNPM_DATASOURCE_URL}
+      SPRING_DATASOURCE_PASSWORD: ${DNPM_MARIADB_USER_PW}
+      SPRING_DATASOURCE_USERNAME: ${DNPM_MARIADB_DB}
+      APP_PSEUDONYMIZE_GPAS_SSLCALOCATION: /workspace/opt/dnpm-processor/ssl/mosaic.crt
+      APP_PSEUDONYMIZE_GPAS_PASSWORD: ${DNPM_PSEUDONYMIZE_GPAS_PASSWORD}
+      APP_PSEUDONYMIZE_GPAS_USERNAME: ${DNPM_PSEUDONYMIZE_GPAS_USERNAME}
+      APP_PSEUDONYMIZE_GPAS_TARGET: ${DNPM_PSEUDONYMIZE_GPAS_TARGET}
+      APP_PSEUDONYMIZE_GPAS_URI: ${DNPM_PSEUDONYMIZE_GPAS_URI}
+      APP_PSEUDONYMIZE_PREFIX: ${DNPM_APP_PSEUDONYMIZE_PREFIX}
+      APP_PSEUDONYMIZER: ${DNPM_PSEUDONYMIZE_GENERATOR}
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+      - /etc/timezone:/etc/timezone:ro
+      #- ${DNPM_TO_SSL_KEYSTORE_LOCATION}:/workspace/opt/dnpm-processor/ssl/keystore.jks:ro
+      #- ${KAFKA_TRUST_STORE_LOCATION}:/workspace/opt/dnpm-processor/ssl/truststore.jks:ro
+      #- ${DNPM_PSEUDONYMIZE_GPAS_SSLCALOCATION}:/workspace/opt/dnpm-processor/ssl/mosaic.crt
+
+    depends_on:
+      - dnpm-monitor-db
+    ports:
+      - "${DNPM_MONITORING_HTTP_PORT:-8080}:8080"
+
+  # todo add volume
+  dnpm-monitor-db:
+    image: mariadb:10
+    environment:
+      MARIADB_DATABASE: ${DNPM_MARIADB_DB}
+      MARIADB_USER: ${DNPM_MARIADB_USER}
+      MARIADB_PASSWORD: ${DNPM_MARIADB_USER_PW}
+      MARIADB_ROOT_PASSWORD: ${DNPM_MARIADB_ROOT_PW}
+    expose:
+      - "3306"
+
+
+

deploy/env-sample.env

@@ -0,0 +1,40 @@
+# monitoring access port
+DNPM_MONITORING_HTTP_PORT=8088
+DNPM_LOG_LEVEL=INFO
+
+# GPAS or BUILDIN
+DNPM_PSEUDONYMIZE_GENERATOR=BUILDIN
+DNPM_APP_PSEUDONYMIZE_PREFIX=ANONYM
+DNPM_PSEUDONYMIZE_GPAS_URI=
+DNPM_PSEUDONYMIZE_GPAS_TARGET=
+DNPM_PSEUDONYMIZE_GPAS_USERNAME=
+DNPM_PSEUDONYMIZE_GPAS_PASSWORD=
+
+# path to ca root cert if needed
+DNPM_PSEUDONYMIZE_GPAS_SSLCALOCATION=
+
+DNPM_MARIADB_DB=dnpm_monitoring
+DNPM_MARIADB_USER=$DNPM_MARIADB_DB
+DNPM_MARIADB_USER_PW=MySuperSecurePassword111
+DNPM_MARIADB_ROOT_PW=MySuperDuperSecurePassword111
+
+# monitoring data db
+DNPM_DATASOURCE_URL=jdbc:mariadb://dnpm-monitor-db:3306/$DNPM_MARIADB_DB
+
+## TARGET SYSTEMS CONFIG
+# in case of direct access to bwhc enter endpoint url here
+DNPM_BWHC_REST_URI=
+
+# produce mtb files to this topic - values 'false' disabling kafka processing
+DNPM_KAFKA_TOPIC=false
+KAFKA_BROKERS=false
+DNPM_KAFKA_SECURITY_PROTOCOL=PLAINTEXT
+
+# here we receive responses from bwhc
+DNPM_KAFKA_RESPONSE_TOPIC=dnpm-response
+DNPM_KAFKA_GROUP_ID=dnpm
+
+# SSL or PLAINTEXT
+DNPM_PROCESSOR_KEY_STORE_PASSWORD=
+DNPM_TO_SSL_KEYSTORE_LOCATION=
+

dev-compose.yml

@@ -1,5 +1,4 @@
 services:
-
   # Note: Make sure, hostname "kafka" points to 127.0.0.1
   # otherwise connection will not be available
   kafka:
@@ -20,6 +19,21 @@ services:
       KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093
       KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
 
+  akhq:
+    image: tchiotludo/akhq:0.21.0
+    environment:
+      AKHQ_CONFIGURATION: |
+        akhq:
+          connections:
+            docker-kafka-server:
+              properties:
+                bootstrap.servers: "kafka:9092"
+              connect:
+                - name: "kafka-connect"
+                  url: "http://kafka-connect:8083"
+    ports:
+      - "8084:8080"
+
   mariadb:
     image: mariadb:10
     ports:

src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java

@@ -22,6 +22,21 @@ package dev.dnpm.etl.processor.pseudonym;
 import ca.uhn.fhir.context.FhirContext;
 import ca.uhn.fhir.parser.IParser;
 import dev.dnpm.etl.processor.config.GPasConfigProperties;
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.ConnectException;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.util.Base64;
+import java.util.HashMap;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManagerFactory;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
 import org.apache.hc.client5.http.impl.classic.HttpClients;
@@ -39,7 +54,11 @@ import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.springframework.http.*;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
 import org.springframework.retry.RetryCallback;
 import org.springframework.retry.RetryContext;
@@ -51,22 +70,6 @@ import org.springframework.retry.support.RetryTemplate;
 import org.springframework.web.client.RestClientException;
 import org.springframework.web.client.RestTemplate;
 
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManagerFactory;
-import java.io.BufferedInputStream;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.net.ConnectException;
-import java.security.KeyManagementException;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.CertificateException;
-import java.security.cert.CertificateFactory;
-import java.security.cert.X509Certificate;
-import java.util.Base64;
-import java.util.HashMap;
-
 public class GpasPseudonymGenerator implements Generator {
 
     private final static FhirContext r4Context = FhirContext.forR4();
@@ -88,12 +91,16 @@ public class GpasPseudonymGenerator implements Generator {
         try {
             if (StringUtils.isNotBlank(gpasCfg.getSslCaLocation())) {
                 customSslContext = getSslContext(gpasCfg.getSslCaLocation());
+                log.debug(String.format("%s has been initialized with SSL certificate %s",
+                    this.getClass().getName(), gpasCfg.getSslCaLocation()));
             }
         } catch (IOException | KeyManagementException | KeyStoreException | CertificateException |
                  NoSuchAlgorithmException e) {
             throw new RuntimeException(e);
         }
 
+        log.debug(String.format("%s has been initialized", this.getClass().getName()));
+
     }
 
     @Override
@@ -115,9 +122,9 @@ public class GpasPseudonymGenerator implements Generator {
         }
 
         final var identifier = (Identifier) parameters.get().getPart().stream()
-                .filter(a -> a.getName().equals("pseudonym"))
+            .filter(a -> a.getName().equals("pseudonym"))
-                .findFirst()
+            .findFirst()
-                .orElseGet(ParametersParameterComponent::new).getValue();
+            .orElseGet(ParametersParameterComponent::new).getValue();
 
         // pseudonym
         return identifier.getSystem() + "|" + identifier.getValue();