From a075f731625db4a790d567de4da7d018c3696131 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Wed, 2 Aug 2023 15:19:38 +0200 Subject: [PATCH 01/13] feat: add Dockerfile for build within docker environment and run application within a container. --- Dockerfile | 36 ++++++++++++++++++++++++++++++++++++ HealthCheck.java | 20 ++++++++++++++++++++ build.gradle.kts | 6 ++++-- 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 Dockerfile create mode 100644 HealthCheck.java diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2d38c6a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +FROM gradle:8.1-jdk17 AS build +WORKDIR /home/gradle/src +ENV GRADLE_USER_HOME /gradle + +COPY build.gradle.kts settings.gradle.kts ./ + +COPY --chown=gradle:gradle . . +RUN gradle build --info && \ + java -Djarmode=layertools -jar build/libs/*.jar extract + +FROM gcr.io/distroless/java17:nonroot +WORKDIR /opt/kafka-streams-template +COPY --from=build /home/gradle/src/dependencies/ ./ +COPY --from=build /home/gradle/src/spring-boot-loader/ ./ +COPY --from=build /home/gradle/src/application/ ./ +COPY HealthCheck.java . + +USER nonroot +ARG GIT_REF="" +ARG GIT_URL="" +ARG BUILD_TIME="" +ARG VERSION=0.0.0 +ENV APP_VERSION=${VERSION} \ + SPRING_PROFILES_ACTIVE="prod" +ENTRYPOINT ["java", "-XX:MaxRAMPercentage=90", "org.springframework.boot.loader.JarLauncher"] + +HEALTHCHECK --interval=25s --timeout=3s --retries=2 CMD ["java", "HealthCheck.java", "||", "exit", "1"] + +LABEL org.opencontainers.image.created=${BUILD_TIME} \ + org.opencontainers.image.authors="Paul-Chrisitan Volkmer, Jakub Lidke" \ + org.opencontainers.image.source=${GIT_URL} \ + org.opencontainers.image.version=${VERSION} \ + org.opencontainers.image.revision=${GIT_REF} \ + org.opencontainers.image.vendor="" \ + org.opencontainers.image.title="etl-processor" \ + org.opencontainers.image.description="Relay application between Onkostar and bwHc for pseudonymization or anonymization of patient data and profide additional monitoring of processed data." diff --git a/HealthCheck.java b/HealthCheck.java new file mode 100644 index 0000000..0aac2a6 --- /dev/null +++ b/HealthCheck.java @@ -0,0 +1,20 @@ +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse.BodyHandlers; + +public class HealthCheck { + + public static void main(String[] args) throws InterruptedException, IOException { + var client = HttpClient.newHttpClient(); + var request = HttpRequest.newBuilder() + .uri(URI.create("http://localhost:8001/actuator/health")) + .header("accept", "application/json") + .build(); + var response = client.send(request, BodyHandlers.ofString()); + if (response.statusCode() != 200 || !response.body().contains("UP")) { + throw new RuntimeException("Healthcheck failed"); + } + } +} \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index eecd959..290362a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,6 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile plugins { - war id("org.springframework.boot") version "3.1.1" id("io.spring.dependency-management") version "1.1.0" kotlin("jvm") version "1.9.0" @@ -28,6 +27,10 @@ repositories { mavenCentral() } +tasks.getByName("jar") { + enabled = false +} + dependencies { implementation("org.jetbrains.kotlin:kotlin-reflect") implementation("org.springframework.boot:spring-boot-starter-thymeleaf") @@ -47,7 +50,6 @@ dependencies { developmentOnly("org.springframework.boot:spring-boot-devtools") developmentOnly("org.springframework.boot:spring-boot-docker-compose") annotationProcessor("org.springframework.boot:spring-boot-configuration-processor") - providedRuntime("org.springframework.boot:spring-boot-starter-tomcat") testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("io.projectreactor:reactor-test") } From 3d2c73ff8f28138f4d377958b779d2b8220659e5 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Thu, 24 Aug 2023 13:01:29 +0200 Subject: [PATCH 02/13] doc: gPas Version requirement added --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index db6ae44..f56a34b 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Diese Anwendung versendet ein bwHC-MTB-File an das bwHC-Backend und pseudonymisi ## Pseudonymisierung der Patienten-ID -Wenn eine URI zu einer gPAS-Instanz angegeben ist, wird diese verwendet. +Wenn eine URI zu einer gPAS-Instanz (Version >= 2023.1.0) angegeben ist, wird diese verwendet. Ist diese nicht gesetzt. wird intern eine Anonymisierung der Patienten-ID vorgenommen. * `APP_PSEUDONYMIZE_PREFIX`: Standortbezogenes Prefix - `UNKNOWN`, wenn nicht gesetzt @@ -43,4 +43,18 @@ Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an ein * `APP_KAFKA_TOPIC`: Zu verwendendes Topic * `APP_KAFKA_SERVERS`: Zu verwendende Kafka-Bootstrap-Server als kommagetrennte Liste -Weitere Einstellungen können über die Parameter von Spring Kafka konfiguriert werden. \ No newline at end of file +Weitere Einstellungen können über die Parameter von Spring Kafka konfiguriert werden. + + +### Docker Image +Bauen eines Docker Images kann wie folgt erzeugt werden: + +```bash +docker build . -t "imageName" +``` + +*Ausführen als Docker Conatiner:* + +```bash +docker compose -f deploy/docker-compose.yaml up -d +``` \ No newline at end of file From 480f165c7b7fb8b80326083af3b04f08d5b2be9b Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Thu, 24 Aug 2023 13:48:46 +0200 Subject: [PATCH 03/13] chore: add deployment docker-compose.yaml and fitting env-sample.env file --- deploy/docker-compose.yaml | 25 ++++++++++++++++++++++ deploy/env-sample.env | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 deploy/docker-compose.yaml create mode 100644 deploy/env-sample.env diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml new file mode 100644 index 0000000..30c647c --- /dev/null +++ b/deploy/docker-compose.yaml @@ -0,0 +1,25 @@ +version: '3.7' + +services: + dnpm-etl-processor: + image: dnpm-elt-processor:latest + env_file: + - ./env-sample.env + depends_on: + - db + networks: + - dnpm_processor + + db: + image: mariadb:10 + environment: + MARIADB_DATABASE: dev + MARIADB_USER: dev + MARIADB_PASSWORD: dev + MARIADB_ROOT_PASSWORD: dev + expose: + - "3306" + networks: + - dnpm_processor +networks: + dnpm_processor: {} diff --git a/deploy/env-sample.env b/deploy/env-sample.env new file mode 100644 index 0000000..fd95b63 --- /dev/null +++ b/deploy/env-sample.env @@ -0,0 +1,44 @@ +# monitoring access port +SERVER_PORT=8000 + +# GPAS or BUILDIN +APP_PSEUDONYMIZE_GENERATOR=BUILDIN +APP_PSEUDONYMIZE_PREFIX= +APP_PSEUDONYMIZE_GPAS_URI= +APP_PSEUDONYMIZE_GPAS_TARGET= +APP_PSEUDONYMIZE_GPAS_USERNAME= +APP_PSEUDONYMIZE_GPAS_PASSWORD= + +# path to ca root cert if needed +APP_PSEUDONYMIZE_GPAS_SSLCALOCATION= + +# monitoring data db +SPRING_DATASOURCE_URL:jdbc:mariadb://db:3306/dev +SPRING_DATASOURCE_PASSWORD: dev +SPRING_DATASOURCE_USERNAME: dev + +## TARGET SYSTEMS CONFIG +# DIRECT BWHC +# in case of direct access to bwhc enter endpoint url here +APP_REST_URI= + +## Apache KAFKA +# list of broker instances +APP_KAFKA_SERVERS= + +# produce mtb files to this topic +APP_KAFKA_TOPIC=mtb-file-json + +# here we receive responses from bwhc +APP_KAFKA_RESPONSE_TOPIC= +APP_KAFKA_GROUP_ID=dnpm + +# SSL or PLAINTEXT +SPRING_KAFKA_SECURITY_PROTOCOL=PLAINTEXT +SPRING_KAFKA_SSL_TRUST-STORE-TYPE=PKCS12 +SPRING_KAFKA_SSL_TRUST-STORE-LOCATION=file://opt/kafka-certs/ca.p12} +SPRING_KAFKA_SSL_TRUST-STORE-PASSWORD= +SPRING_KAFKA_SSL_KEY-STORE-TYPE=PKCS12 +SPRING_KAFKA_SSL_KEY-STORE-LOCATION=file://opt/kafka-certs/user.p12} +SPRING_KAFKA_SSL_KEY-STORE-PASSWORD= +SPRING_KAFKA_PRODUCER_COMPRESSION-TYPE: gzip From bbea48322fa5a24ff61eef25ed84f4966bf49c42 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Fri, 25 Aug 2023 12:50:29 +0200 Subject: [PATCH 04/13] chore: added deployment port mapping for monitoring access --- .gitignore | 1 + README.md | 34 +++++++++++++++++++++++----------- deploy/docker-compose.yaml | 4 +++- deploy/env-sample.env | 2 +- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 4ae22a7..2be9b02 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ out/ ### VS Code ### .vscode/ /dev/gpas* +/deploy/.env diff --git a/README.md b/README.md index f56a34b..ce086d4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # ETL-Processor for bwHC data -Diese Anwendung versendet ein bwHC-MTB-File an das bwHC-Backend und pseudonymisiert die Patienten-ID. +Diese Anwendung versendet ein bwHC-MTB-File an das bwHC-Backend und pseudonymisiert die +Patienten-ID. ## Pseudonymisierung der Patienten-ID @@ -12,41 +13,49 @@ Ist diese nicht gesetzt. wird intern eine Anonymisierung der Patienten-ID vorgen ### Eingebaute Pseudonymisierung -Wurde keine oder die Verwendung der eingebauten Pseudonymisierung konfiguriert, so wird für die Patienten-ID der -entsprechende SHA-256-Hash gebildet und Base64-codiert - hier ohne endende "=" - zuzüglich des konfigurierten Prefixes +Wurde keine oder die Verwendung der eingebauten Pseudonymisierung konfiguriert, so wird für die +Patienten-ID der +entsprechende SHA-256-Hash gebildet und Base64-codiert - hier ohne endende "=" - zuzüglich des +konfigurierten Prefixes als Patienten-Pseudonym verwendet. ### Pseudonymisierung mit gPAS Wurde die Verwendung von gPAS konfiguriert, so sind weitere Angaben zu konfigurieren. -* `APP_PSEUDONYMIZE_GPAS_URI`: URI der gPAS-Instanz inklusive Endpoint (z.B. `http://localhost:8080/ttp-fhir/fhir/gpas/$pseudonymizeAllowCreate`) +* `APP_PSEUDONYMIZE_GPAS_URI`: URI der gPAS-Instanz inklusive Endpoint ( + z.B. `http://localhost:8080/ttp-fhir/fhir/gpas/$pseudonymizeAllowCreate`) * `APP_PSEUDONYMIZE_GPAS_TARGET`: gPas Domänenname * `APP_PSEUDONYMIZE_GPAS_USERNAME`: gPas Basic-Auth Benutzername * `APP_PSEUDONYMIZE_GPAS_PASSWORD`: gPas Basic-Auth Passwort -* `APP_PSEUDONYMIZE_GPAS_SSLCALOCATION`: Root Zertifikat für gPas, falls es dediziert hinzugefügt werden muss. +* `APP_PSEUDONYMIZE_GPAS_SSLCALOCATION`: Root Zertifikat für gPas, falls es dediziert hinzugefügt + werden muss. ## Mögliche Endpunkte -Für REST-Requests als auch (parallel) zur Nutzung von Kafka-Topics können Endpunkte konfiguriert werden. +Für REST-Requests als auch (parallel) zur Nutzung von Kafka-Topics können Endpunkte konfiguriert +werden. ### REST -Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an das bwHC-Backend gesendet wird: +Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an das bwHC-Backend +gesendet wird: -* `APP_REST_URI`: URI der zu benutzenden API der bwHC-Backend-Instanz. z.B.: `http://localhost:9000/bwhc/etl/api` +* `APP_REST_URI`: URI der zu benutzenden API der bwHC-Backend-Instanz. + z.B.: `http://localhost:9000/bwhc/etl/api` ### Kafka-Topics -Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an ein Kafka-Topic übermittelt wird: +Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an ein Kafka-Topic +übermittelt wird: * `APP_KAFKA_TOPIC`: Zu verwendendes Topic * `APP_KAFKA_SERVERS`: Zu verwendende Kafka-Bootstrap-Server als kommagetrennte Liste Weitere Einstellungen können über die Parameter von Spring Kafka konfiguriert werden. - ### Docker Image + Bauen eines Docker Images kann wie folgt erzeugt werden: ```bash @@ -54,7 +63,10 @@ docker build . -t "imageName" ``` *Ausführen als Docker Conatiner:* +Wenn gewünscht, Änderungen in der `env` vornehmen. Beachten, dass *MONITORING_HTTP_PORT* über +Host-Umgebung gesetzt werden muss (z.B. .env oder Parameter --env-file ) ```bash -docker compose -f deploy/docker-compose.yaml up -d +cd ./deploy +docker compose up -d ``` \ No newline at end of file diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml index 30c647c..8dccd4c 100644 --- a/deploy/docker-compose.yaml +++ b/deploy/docker-compose.yaml @@ -1,4 +1,4 @@ -version: '3.7' + services: dnpm-etl-processor: @@ -9,6 +9,8 @@ services: - db networks: - dnpm_processor + ports: + - "${MONITORING_HTTP_PORT:-8080}:8080" db: image: mariadb:10 diff --git a/deploy/env-sample.env b/deploy/env-sample.env index fd95b63..1589096 100644 --- a/deploy/env-sample.env +++ b/deploy/env-sample.env @@ -1,5 +1,5 @@ # monitoring access port -SERVER_PORT=8000 +MONITORING_HTTP_PORT=8088 # GPAS or BUILDIN APP_PSEUDONYMIZE_GENERATOR=BUILDIN From aa40da49957ebad06fff277d6d9937fc370ded44 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Fri, 25 Aug 2023 13:11:10 +0200 Subject: [PATCH 05/13] chore: dev kafka is available via localhost, now. --- dev-compose.yml | 7 +++++-- src/main/resources/application-dev.yml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dev-compose.yml b/dev-compose.yml index 8f0780f..647b1aa 100644 --- a/dev-compose.yml +++ b/dev-compose.yml @@ -11,8 +11,11 @@ services: ALLOW_PLAINTEXT_LISTENER: "yes" KAFKA_CFG_NODE_ID: "0" KAFKA_CFG_PROCESS_ROLES: "controller,broker" - KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CFG_LISTENERS: PLAINTEXT://:9094,CONTROLLER://:9093,EXTERNAL://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9094,EXTERNAL://localhost:9092 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: PLAINTEXT + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093 KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index a60cd8a..3134643 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -12,7 +12,7 @@ app: kafka: topic: test response-topic: test_response - servers: kafka:9092 + servers: localhost:9092 server: port: 8000 From 9643c80cc520584291f72a126b1f9b64f927dd5b Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Fri, 25 Aug 2023 13:39:42 +0200 Subject: [PATCH 06/13] build: locally build docker image has license entry,now --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 2d38c6a..f5f6029 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,5 +32,6 @@ LABEL org.opencontainers.image.created=${BUILD_TIME} \ org.opencontainers.image.version=${VERSION} \ org.opencontainers.image.revision=${GIT_REF} \ org.opencontainers.image.vendor="" \ + org.opencontainers.image.license="AGPLv3" \ org.opencontainers.image.title="etl-processor" \ org.opencontainers.image.description="Relay application between Onkostar and bwHc for pseudonymization or anonymization of patient data and profide additional monitoring of processed data." From 01ff53ab235488cb23e8aa335be7bf58c0e7bf53 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Fri, 25 Aug 2023 13:42:07 +0200 Subject: [PATCH 07/13] chore: deployment environment has maria db entries --- deploy/docker-compose.yaml | 12 ++++++------ deploy/env-sample.env | 19 ++++++++++++------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml index 8dccd4c..a1090ec 100644 --- a/deploy/docker-compose.yaml +++ b/deploy/docker-compose.yaml @@ -15,12 +15,12 @@ services: db: image: mariadb:10 environment: - MARIADB_DATABASE: dev - MARIADB_USER: dev - MARIADB_PASSWORD: dev - MARIADB_ROOT_PASSWORD: dev - expose: - - "3306" + MARIADB_DATABASE: ${MARIADB_DB} + MARIADB_USER: ${MARIADB_USER} + MARIADB_PASSWORD: ${MARIADB_USER_PW} + MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PW} + ports: + - "3306:3306" networks: - dnpm_processor networks: diff --git a/deploy/env-sample.env b/deploy/env-sample.env index 1589096..63fde20 100644 --- a/deploy/env-sample.env +++ b/deploy/env-sample.env @@ -12,10 +12,15 @@ APP_PSEUDONYMIZE_GPAS_PASSWORD= # path to ca root cert if needed APP_PSEUDONYMIZE_GPAS_SSLCALOCATION= +MARIADB_DB=dnpm_monitoring +MARIADB_USER=$MARIADB_DB +MARIADB_USER_PW=MySuperSecurePassword111 +MARIADB_ROOT_PW=MySuperDuperSecurePassword111 + # monitoring data db -SPRING_DATASOURCE_URL:jdbc:mariadb://db:3306/dev -SPRING_DATASOURCE_PASSWORD: dev -SPRING_DATASOURCE_USERNAME: dev +SPRING_DATASOURCE_URL:jdbc:mariadb://db:3306/$MARIADB_DB +SPRING_DATASOURCE_PASSWORD: $MARIADB_USER_PW +SPRING_DATASOURCE_USERNAME: dnpm_monitoring ## TARGET SYSTEMS CONFIG # DIRECT BWHC @@ -24,14 +29,14 @@ APP_REST_URI= ## Apache KAFKA # list of broker instances -APP_KAFKA_SERVERS= +#APP_KAFKA_SERVERS= # produce mtb files to this topic -APP_KAFKA_TOPIC=mtb-file-json +#APP_KAFKA_TOPIC=mtb-file-json # here we receive responses from bwhc -APP_KAFKA_RESPONSE_TOPIC= -APP_KAFKA_GROUP_ID=dnpm +#APP_KAFKA_RESPONSE_TOPIC= +#APP_KAFKA_GROUP_ID=dnpm # SSL or PLAINTEXT SPRING_KAFKA_SECURITY_PROTOCOL=PLAINTEXT From 532254593fbc1a8217a276fe94f7d9fcd87da6b5 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Mon, 28 Aug 2023 12:47:09 +0200 Subject: [PATCH 08/13] test: * added additional external host 'localhost', now we can connect without additional host alias. * added akhq to dev-compose.yml --- dev-compose.yml | 24 ++++++++++++++++++++---- src/main/resources/application-dev.yml | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/dev-compose.yml b/dev-compose.yml index 647b1aa..08db2e3 100644 --- a/dev-compose.yml +++ b/dev-compose.yml @@ -1,5 +1,4 @@ services: - # Note: Make sure, hostname "kafka" points to 127.0.0.1 # otherwise connection will not be available kafka: @@ -7,18 +6,34 @@ services: hostname: kafka ports: - "9092:9092" + - "9094:9094" environment: ALLOW_PLAINTEXT_LISTENER: "yes" KAFKA_CFG_NODE_ID: "0" KAFKA_CFG_PROCESS_ROLES: "controller,broker" - KAFKA_CFG_LISTENERS: PLAINTEXT://:9094,CONTROLLER://:9093,EXTERNAL://:9092 - KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9094,EXTERNAL://localhost:9092 + KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 + KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094 KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT KAFKA_CFG_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093 KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + akhq: + image: tchiotludo/akhq:0.21.0 + environment: + AKHQ_CONFIGURATION: | + akhq: + connections: + docker-kafka-server: + properties: + bootstrap.servers: "kafka:9092" + connect: + - name: "kafka-connect" + url: "http://kafka-connect:8083" + ports: + - "8084:8080" + mariadb: image: mariadb:10 ports: @@ -28,6 +43,7 @@ services: MARIADB_USER: dev MARIADB_PASSWORD: dev MARIADB_ROOT_PASSWORD: dev + # postgres: # image: postgres:alpine # ports: @@ -35,4 +51,4 @@ services: # environment: # POSTGRES_DB: dev # POSTGRES_USER: dev -# POSTGRES_PASSWORD: dev +# POSTGRES_PASSWORD: dev \ No newline at end of file diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 3134643..3a055e8 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -12,7 +12,7 @@ app: kafka: topic: test response-topic: test_response - servers: localhost:9092 + servers: localhost:9094 server: port: 8000 From 25143745c4691646112266006d68161bcb359bdd Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Mon, 28 Aug 2023 12:54:14 +0200 Subject: [PATCH 09/13] chore: added deployment docker-compose.yaml and env-sample.env added. --- README.md | 7 +++++-- deploy/docker-compose.yaml | 36 +++++++++++++++++++++++++++--------- deploy/env-sample.env | 31 ++++++++++++++++--------------- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b743d92..8290dd1 100644 --- a/README.md +++ b/README.md @@ -129,11 +129,14 @@ docker build . -t "imageName" ## Deployment *Ausführen als Docker Conatiner:* -Wenn gewünscht, Änderungen in der `env` vornehmen. Beachten, dass *MONITORING_HTTP_PORT* über -Host-Umgebung gesetzt werden muss (z.B. .env oder Parameter --env-file ) ```bash cd ./deploy +cp env-sample.env .env +``` +Wenn gewünscht, Änderungen in der `.env` vornehmen. + +```bash docker compose up -d ``` diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml index a1090ec..5e9d8ef 100644 --- a/deploy/docker-compose.yaml +++ b/deploy/docker-compose.yaml @@ -2,13 +2,34 @@ services: dnpm-etl-processor: - image: dnpm-elt-processor:latest - env_file: - - ./env-sample.env + image: ghcr.io/ccc-mf/etl-processor:latest + environment: + SPRING_KAFKA_SECURITY_PROTOCOL: ${SPRING_KAFKA_SECURITY_PROTOCOL} + SPRING_KAFKA_SSL_TRUST-STORE-TYPE: ${SPRING_KAFKA_SSL_TRUST_STORE_TYPE} + SPRING_KAFKA_SSL_TRUST-STORE-LOCATION: ${SPRING_KAFKA_SSL_TRUST_STORE_LOCATION} + SPRING_KAFKA_SSL_TRUST-STORE-PASSWORD: ${SPRING_KAFKA_SSL_TRUST_STORE_PASSWORD} + SPRING_KAFKA_SSL_KEY-STORE-TYPE: ${SPRING_KAFKA_SSL_KEY_STORE_TYPE} + SPRING_KAFKA_SSL_KEY-STORE-LOCATION: ${KAFKA_KEY_STORE_LOCATION} + SPRING_KAFKA_SSL_KEY-STORE-PASSWORD: ${DNPM_PROCESSOR_KEY_STORE_PASSWORD} + SPRING_KAFKA_PRODUCER_COMPRESSION-TYPE: ${SPRING_KAFKA_PRODUCER_COMPRESSION_TYPE} + APP_KAFKA_TOPIC: ${APP_KAFKA_TOPIC} + APP_KAFKA_SERVERS: ${APP_KAFKA_SERVERS} + APP_KAFKA_GROUP_ID: ${APP_KAFKA_GROUP_ID} + APP_KAFKA_RESPONSE_TOPIC: ${APP_KAFKA_RESPONSE_TOPIC} + APP_REST_URI: ${APP_REST_URI} + SPRING_DATASOURCE_URL: ${SPRING_DATASOURCE_URL} + SPRING_DATASOURCE_PASSWORD: ${SPRING_DATASOURCE_PASSWORD} + SPRING_DATASOURCE_USERNAME: ${SPRING_DATASOURCE_USERNAME} + APP_PSEUDONYMIZE_GPAS_SSLCALOCATION: ${APP_PSEUDONYMIZE_GPAS_SSLCALOCATION} + APP_PSEUDONYMIZE_GPAS_PASSWORD: ${APP_PSEUDONYMIZE_GPAS_PASSWORD} + APP_PSEUDONYMIZE_GPAS_USERNAME: ${APP_PSEUDONYMIZE_GPAS_USERNAME} + APP_PSEUDONYMIZE_GPAS_TARGET: ${APP_PSEUDONYMIZE_GPAS_TARGET} + APP_PSEUDONYMIZE_GPAS_URI: ${APP_PSEUDONYMIZE_GPAS_URI} + APP_PSEUDONYMIZE_PREFIX: ${APP_PSEUDONYMIZE_PREFIX} + APP_PSEUDONYMIZE_GENERATOR: ${APP_PSEUDONYMIZE_GENERATOR} + depends_on: - db - networks: - - dnpm_processor ports: - "${MONITORING_HTTP_PORT:-8080}:8080" @@ -21,7 +42,4 @@ services: MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PW} ports: - "3306:3306" - networks: - - dnpm_processor -networks: - dnpm_processor: {} + diff --git a/deploy/env-sample.env b/deploy/env-sample.env index 63fde20..7399f7b 100644 --- a/deploy/env-sample.env +++ b/deploy/env-sample.env @@ -18,32 +18,33 @@ MARIADB_USER_PW=MySuperSecurePassword111 MARIADB_ROOT_PW=MySuperDuperSecurePassword111 # monitoring data db -SPRING_DATASOURCE_URL:jdbc:mariadb://db:3306/$MARIADB_DB -SPRING_DATASOURCE_PASSWORD: $MARIADB_USER_PW -SPRING_DATASOURCE_USERNAME: dnpm_monitoring +SPRING_DATASOURCE_URL=jdbc:mariadb://db:3306/$MARIADB_DB +SPRING_DATASOURCE_PASSWORD=$MARIADB_USER_PW +SPRING_DATASOURCE_USERNAME=dnpm_monitoring ## TARGET SYSTEMS CONFIG # DIRECT BWHC # in case of direct access to bwhc enter endpoint url here APP_REST_URI= -## Apache KAFKA +## Apache KAFKA if APP_KAFKA_SERVERS and APP_KAFKA_TOPIC have value 'false' kafka support is disabled # list of broker instances -#APP_KAFKA_SERVERS= +APP_KAFKA_SERVERS=false # produce mtb files to this topic -#APP_KAFKA_TOPIC=mtb-file-json +APP_KAFKA_TOPIC=false # here we receive responses from bwhc -#APP_KAFKA_RESPONSE_TOPIC= -#APP_KAFKA_GROUP_ID=dnpm +APP_KAFKA_RESPONSE_TOPIC=dnpm-response +APP_KAFKA_GROUP_ID=dnpm # SSL or PLAINTEXT SPRING_KAFKA_SECURITY_PROTOCOL=PLAINTEXT -SPRING_KAFKA_SSL_TRUST-STORE-TYPE=PKCS12 -SPRING_KAFKA_SSL_TRUST-STORE-LOCATION=file://opt/kafka-certs/ca.p12} -SPRING_KAFKA_SSL_TRUST-STORE-PASSWORD= -SPRING_KAFKA_SSL_KEY-STORE-TYPE=PKCS12 -SPRING_KAFKA_SSL_KEY-STORE-LOCATION=file://opt/kafka-certs/user.p12} -SPRING_KAFKA_SSL_KEY-STORE-PASSWORD= -SPRING_KAFKA_PRODUCER_COMPRESSION-TYPE: gzip +SPRING_KAFKA_SSL_TRUST_STORE_TYPE=PKCS12 +SPRING_KAFKA_SSL_TRUST_STORE_LOCATION=file://opt/kafka-certs/ca.p12 +SPRING_KAFKA_SSL_TRUST_STORE_PASSWORD= +SPRING_KAFKA_SSL_KEY_STORE_TYPE=PKCS12 +SPRING_KAFKA_SSL_KEY_STORE_LOCATION=file://opt/kafka-certs/user.p12 +DNPM_PROCESSOR_KEY_STORE_PASSWORD= +SPRING_KAFKA_PRODUCER_COMPRESSION_TYPE=gzip +KAFKA_KEY_STORE_LOCATION=file://opt/kafka-certs/user.p12 From 635985bfd1cab27cc21710815fa237e04954dea2 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Mon, 28 Aug 2023 14:27:28 +0200 Subject: [PATCH 10/13] chore: remove previous build via Dockerfile. Fix security issue: CVE-2023-34453, CVE-2023-34454, CVE-2023-34455, CVE-2022-1471 --- Dockerfile | 37 ------------------------------------- build.gradle.kts | 21 ++++++++++++++------- 2 files changed, 14 insertions(+), 44 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f5f6029..0000000 --- a/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM gradle:8.1-jdk17 AS build -WORKDIR /home/gradle/src -ENV GRADLE_USER_HOME /gradle - -COPY build.gradle.kts settings.gradle.kts ./ - -COPY --chown=gradle:gradle . . -RUN gradle build --info && \ - java -Djarmode=layertools -jar build/libs/*.jar extract - -FROM gcr.io/distroless/java17:nonroot -WORKDIR /opt/kafka-streams-template -COPY --from=build /home/gradle/src/dependencies/ ./ -COPY --from=build /home/gradle/src/spring-boot-loader/ ./ -COPY --from=build /home/gradle/src/application/ ./ -COPY HealthCheck.java . - -USER nonroot -ARG GIT_REF="" -ARG GIT_URL="" -ARG BUILD_TIME="" -ARG VERSION=0.0.0 -ENV APP_VERSION=${VERSION} \ - SPRING_PROFILES_ACTIVE="prod" -ENTRYPOINT ["java", "-XX:MaxRAMPercentage=90", "org.springframework.boot.loader.JarLauncher"] - -HEALTHCHECK --interval=25s --timeout=3s --retries=2 CMD ["java", "HealthCheck.java", "||", "exit", "1"] - -LABEL org.opencontainers.image.created=${BUILD_TIME} \ - org.opencontainers.image.authors="Paul-Chrisitan Volkmer, Jakub Lidke" \ - org.opencontainers.image.source=${GIT_URL} \ - org.opencontainers.image.version=${VERSION} \ - org.opencontainers.image.revision=${GIT_REF} \ - org.opencontainers.image.vendor="" \ - org.opencontainers.image.license="AGPLv3" \ - org.opencontainers.image.title="etl-processor" \ - org.opencontainers.image.description="Relay application between Onkostar and bwHc for pseudonymization or anonymization of patient data and profide additional monitoring of processed data." diff --git a/build.gradle.kts b/build.gradle.kts index d8389e6..61ca3ed 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -3,8 +3,9 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import org.springframework.boot.gradle.tasks.bundling.BootBuildImage plugins { - id("org.springframework.boot") version "3.1.1" - id("io.spring.dependency-management") version "1.1.0" + war + id("org.springframework.boot") version "3.1.2" + id("io.spring.dependency-management") version "1.1.3" kotlin("jvm") version "1.9.0" kotlin("plugin.spring") version "1.9.0" } @@ -48,17 +49,22 @@ repositories { mavenCentral() } -tasks.getByName("jar") { - enabled = false -} - dependencies { implementation("org.jetbrains.kotlin:kotlin-reflect") implementation("org.springframework.boot:spring-boot-starter-thymeleaf") implementation("org.springframework.boot:spring-boot-starter-web") implementation("org.springframework.boot:spring-boot-starter-data-jdbc") implementation("com.fasterxml.jackson.module:jackson-module-kotlin") - implementation("org.springframework.kafka:spring-kafka") + implementation("org.springframework.kafka:spring-kafka"){ + // CVE-2023-34453, CVE-2023-34454, CVE-2023-34455 + exclude( "org.xerial.snappy:snappy-java") + // CVE-2022-1471 + exclude("org.yaml:snakeyaml") + } + // fixes CVE-2023-34453, CVE-2023-34454, CVE-2023-34455 + implementation("org.xerial.snappy:snappy-java:1.1.10.3") + // fix CVE-2022-1471 + implementation("org.yaml:snakeyaml:2.1") implementation("org.flywaydb:flyway-mysql") implementation("commons-codec:commons-codec") implementation("io.projectreactor.kotlin:reactor-kotlin-extensions") @@ -71,6 +77,7 @@ dependencies { developmentOnly("org.springframework.boot:spring-boot-devtools") developmentOnly("org.springframework.boot:spring-boot-docker-compose") annotationProcessor("org.springframework.boot:spring-boot-configuration-processor") + providedRuntime("org.springframework.boot:spring-boot-starter-tomcat") testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation("io.projectreactor:reactor-test") testImplementation("org.mockito.kotlin:mockito-kotlin:${versions["mockito-kotlin"]}") From e24be0d32592623ae2fc0e3d7f3618437036d72a Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Wed, 30 Aug 2023 11:50:24 +0200 Subject: [PATCH 11/13] chore: cleanup deployment docker-compose.yaml and env-sample.env. added 'DNPM' prefix for better integration into productive environment. --- README.md | 4 +-- deploy/docker-compose.yaml | 72 ++++++++++++++++++++++---------------- deploy/env-sample.env | 56 ++++++++++++----------------- 3 files changed, 66 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 8290dd1..908b1de 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ als Patienten-Pseudonym verwendet. Wurde die Verwendung von gPAS konfiguriert, so sind weitere Angaben zu konfigurieren. * `APP_PSEUDONYMIZE_GPAS_URI`: URI der gPAS-Instanz inklusive Endpoint ( - z.B. `http://localhost:8080/ttp-fhir/fhir/gpas/$pseudonymizeAllowCreate`) + z.B. `http://localhost:8080/ttp-fhir/fhir/gpas/$$pseudonymizeAllowCreate`) * `APP_PSEUDONYMIZE_GPAS_TARGET`: gPas Domänenname * `APP_PSEUDONYMIZE_GPAS_USERNAME`: gPas Basic-Auth Benutzername * `APP_PSEUDONYMIZE_GPAS_PASSWORD`: gPas Basic-Auth Passwort @@ -124,7 +124,7 @@ Diese Anwendung ist auch als Docker-Image verfügbar: https://github.com/CCC-MF/ ### Images lokal bauen ```bash -docker build . -t "imageName" +./gradlew bootBuildImage ``` ## Deployment diff --git a/deploy/docker-compose.yaml b/deploy/docker-compose.yaml index 5e9d8ef..d575d09 100644 --- a/deploy/docker-compose.yaml +++ b/deploy/docker-compose.yaml @@ -4,42 +4,52 @@ services: dnpm-etl-processor: image: ghcr.io/ccc-mf/etl-processor:latest environment: - SPRING_KAFKA_SECURITY_PROTOCOL: ${SPRING_KAFKA_SECURITY_PROTOCOL} - SPRING_KAFKA_SSL_TRUST-STORE-TYPE: ${SPRING_KAFKA_SSL_TRUST_STORE_TYPE} - SPRING_KAFKA_SSL_TRUST-STORE-LOCATION: ${SPRING_KAFKA_SSL_TRUST_STORE_LOCATION} - SPRING_KAFKA_SSL_TRUST-STORE-PASSWORD: ${SPRING_KAFKA_SSL_TRUST_STORE_PASSWORD} - SPRING_KAFKA_SSL_KEY-STORE-TYPE: ${SPRING_KAFKA_SSL_KEY_STORE_TYPE} - SPRING_KAFKA_SSL_KEY-STORE-LOCATION: ${KAFKA_KEY_STORE_LOCATION} + LOGGING_LEVEL_DEV: ${DNPM_LOG_LEVEL:-INFO} + SPRING_KAFKA_SECURITY_PROTOCOL: ${DNPM_KAFKA_SECURITY_PROTOCOL:-SSL} + SPRING_KAFKA_SSL_TRUST-STORE-TYPE: PKCS12 + SPRING_KAFKA_SSL_TRUST-STORE-LOCATION: /opt/dnpm-processor/ssl/truststore.jks + SPRING_KAFKA_SSL_TRUST-STORE-PASSWORD: ${KAFKA_TRUST_STORE_PASSWORD} + SPRING_KAFKA_SSL_KEY-STORE-TYPE: PKCS12 + SPRING_KAFKA_SSL_KEY-STORE-LOCATION: /opt/dnpm-processor/ssl/keystore.jks SPRING_KAFKA_SSL_KEY-STORE-PASSWORD: ${DNPM_PROCESSOR_KEY_STORE_PASSWORD} - SPRING_KAFKA_PRODUCER_COMPRESSION-TYPE: ${SPRING_KAFKA_PRODUCER_COMPRESSION_TYPE} - APP_KAFKA_TOPIC: ${APP_KAFKA_TOPIC} - APP_KAFKA_SERVERS: ${APP_KAFKA_SERVERS} - APP_KAFKA_GROUP_ID: ${APP_KAFKA_GROUP_ID} - APP_KAFKA_RESPONSE_TOPIC: ${APP_KAFKA_RESPONSE_TOPIC} - APP_REST_URI: ${APP_REST_URI} - SPRING_DATASOURCE_URL: ${SPRING_DATASOURCE_URL} - SPRING_DATASOURCE_PASSWORD: ${SPRING_DATASOURCE_PASSWORD} - SPRING_DATASOURCE_USERNAME: ${SPRING_DATASOURCE_USERNAME} - APP_PSEUDONYMIZE_GPAS_SSLCALOCATION: ${APP_PSEUDONYMIZE_GPAS_SSLCALOCATION} - APP_PSEUDONYMIZE_GPAS_PASSWORD: ${APP_PSEUDONYMIZE_GPAS_PASSWORD} - APP_PSEUDONYMIZE_GPAS_USERNAME: ${APP_PSEUDONYMIZE_GPAS_USERNAME} - APP_PSEUDONYMIZE_GPAS_TARGET: ${APP_PSEUDONYMIZE_GPAS_TARGET} - APP_PSEUDONYMIZE_GPAS_URI: ${APP_PSEUDONYMIZE_GPAS_URI} - APP_PSEUDONYMIZE_PREFIX: ${APP_PSEUDONYMIZE_PREFIX} - APP_PSEUDONYMIZE_GENERATOR: ${APP_PSEUDONYMIZE_GENERATOR} + SPRING_KAFKA_PRODUCER_COMPRESSION-TYPE: gzip + APP_KAFKA_TOPIC: ${DNPM_KAFKA_TOPIC} + APP_KAFKA_SERVERS: ${KAFKA_BROKERS} + APP_KAFKA_GROUP_ID: ${DNPM_KAFKA_GROUP_ID} + APP_KAFKA_RESPONSE_TOPIC: ${DNPM_KAFKA_RESPONSE_TOPIC} + APP_REST_URI: ${DNPM_BWHC_REST_URI} + SPRING_DATASOURCE_URL: ${DNPM_DATASOURCE_URL} + SPRING_DATASOURCE_PASSWORD: ${DNPM_MARIADB_USER_PW} + SPRING_DATASOURCE_USERNAME: ${DNPM_MARIADB_DB} + APP_PSEUDONYMIZE_GPAS_SSLCALOCATION: /workspace/opt/dnpm-processor/ssl/mosaic.crt + APP_PSEUDONYMIZE_GPAS_PASSWORD: ${DNPM_PSEUDONYMIZE_GPAS_PASSWORD} + APP_PSEUDONYMIZE_GPAS_USERNAME: ${DNPM_PSEUDONYMIZE_GPAS_USERNAME} + APP_PSEUDONYMIZE_GPAS_TARGET: ${DNPM_PSEUDONYMIZE_GPAS_TARGET} + APP_PSEUDONYMIZE_GPAS_URI: ${DNPM_PSEUDONYMIZE_GPAS_URI} + APP_PSEUDONYMIZE_PREFIX: ${DNPM_APP_PSEUDONYMIZE_PREFIX} + APP_PSEUDONYMIZER: ${DNPM_PSEUDONYMIZE_GENERATOR} + volumes: + - /etc/localtime:/etc/localtime:ro + - /etc/timezone:/etc/timezone:ro + #- ${DNPM_TO_SSL_KEYSTORE_LOCATION}:/workspace/opt/dnpm-processor/ssl/keystore.jks:ro + #- ${KAFKA_TRUST_STORE_LOCATION}:/workspace/opt/dnpm-processor/ssl/truststore.jks:ro + #- ${DNPM_PSEUDONYMIZE_GPAS_SSLCALOCATION}:/workspace/opt/dnpm-processor/ssl/mosaic.crt depends_on: - - db + - dnpm-monitor-db ports: - - "${MONITORING_HTTP_PORT:-8080}:8080" + - "${DNPM_MONITORING_HTTP_PORT:-8080}:8080" - db: + # todo add volume + dnpm-monitor-db: image: mariadb:10 environment: - MARIADB_DATABASE: ${MARIADB_DB} - MARIADB_USER: ${MARIADB_USER} - MARIADB_PASSWORD: ${MARIADB_USER_PW} - MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PW} - ports: - - "3306:3306" + MARIADB_DATABASE: ${DNPM_MARIADB_DB} + MARIADB_USER: ${DNPM_MARIADB_USER} + MARIADB_PASSWORD: ${DNPM_MARIADB_USER_PW} + MARIADB_ROOT_PASSWORD: ${DNPM_MARIADB_ROOT_PW} + expose: + - "3306" + + diff --git a/deploy/env-sample.env b/deploy/env-sample.env index 7399f7b..998400a 100644 --- a/deploy/env-sample.env +++ b/deploy/env-sample.env @@ -1,50 +1,40 @@ # monitoring access port -MONITORING_HTTP_PORT=8088 +DNPM_MONITORING_HTTP_PORT=8088 +DNPM_LOG_LEVEL=INFO # GPAS or BUILDIN -APP_PSEUDONYMIZE_GENERATOR=BUILDIN -APP_PSEUDONYMIZE_PREFIX= -APP_PSEUDONYMIZE_GPAS_URI= -APP_PSEUDONYMIZE_GPAS_TARGET= -APP_PSEUDONYMIZE_GPAS_USERNAME= -APP_PSEUDONYMIZE_GPAS_PASSWORD= +DNPM_PSEUDONYMIZE_GENERATOR=BUILDIN +DNPM_APP_PSEUDONYMIZE_PREFIX=ANONYM +DNPM_PSEUDONYMIZE_GPAS_URI= +DNPM_PSEUDONYMIZE_GPAS_TARGET= +DNPM_PSEUDONYMIZE_GPAS_USERNAME= +DNPM_PSEUDONYMIZE_GPAS_PASSWORD= # path to ca root cert if needed -APP_PSEUDONYMIZE_GPAS_SSLCALOCATION= +DNPM_PSEUDONYMIZE_GPAS_SSLCALOCATION= -MARIADB_DB=dnpm_monitoring -MARIADB_USER=$MARIADB_DB -MARIADB_USER_PW=MySuperSecurePassword111 -MARIADB_ROOT_PW=MySuperDuperSecurePassword111 +DNPM_MARIADB_DB=dnpm_monitoring +DNPM_MARIADB_USER=$DNPM_MARIADB_DB +DNPM_MARIADB_USER_PW=MySuperSecurePassword111 +DNPM_MARIADB_ROOT_PW=MySuperDuperSecurePassword111 # monitoring data db -SPRING_DATASOURCE_URL=jdbc:mariadb://db:3306/$MARIADB_DB -SPRING_DATASOURCE_PASSWORD=$MARIADB_USER_PW -SPRING_DATASOURCE_USERNAME=dnpm_monitoring +DNPM_DATASOURCE_URL=jdbc:mariadb://dnpm-monitor-db:3306/$DNPM_MARIADB_DB ## TARGET SYSTEMS CONFIG -# DIRECT BWHC # in case of direct access to bwhc enter endpoint url here -APP_REST_URI= +DNPM_BWHC_REST_URI= -## Apache KAFKA if APP_KAFKA_SERVERS and APP_KAFKA_TOPIC have value 'false' kafka support is disabled -# list of broker instances -APP_KAFKA_SERVERS=false - -# produce mtb files to this topic -APP_KAFKA_TOPIC=false +# produce mtb files to this topic - values 'false' disabling kafka processing +DNPM_KAFKA_TOPIC=false +KAFKA_BROKERS=false +DNPM_KAFKA_SECURITY_PROTOCOL=PLAINTEXT # here we receive responses from bwhc -APP_KAFKA_RESPONSE_TOPIC=dnpm-response -APP_KAFKA_GROUP_ID=dnpm +DNPM_KAFKA_RESPONSE_TOPIC=dnpm-response +DNPM_KAFKA_GROUP_ID=dnpm # SSL or PLAINTEXT -SPRING_KAFKA_SECURITY_PROTOCOL=PLAINTEXT -SPRING_KAFKA_SSL_TRUST_STORE_TYPE=PKCS12 -SPRING_KAFKA_SSL_TRUST_STORE_LOCATION=file://opt/kafka-certs/ca.p12 -SPRING_KAFKA_SSL_TRUST_STORE_PASSWORD= -SPRING_KAFKA_SSL_KEY_STORE_TYPE=PKCS12 -SPRING_KAFKA_SSL_KEY_STORE_LOCATION=file://opt/kafka-certs/user.p12 DNPM_PROCESSOR_KEY_STORE_PASSWORD= -SPRING_KAFKA_PRODUCER_COMPRESSION_TYPE=gzip -KAFKA_KEY_STORE_LOCATION=file://opt/kafka-certs/user.p12 +DNPM_TO_SSL_KEYSTORE_LOCATION= + From edafe30a4bd23a8a8a16c7612d6fdc49d3208829 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Wed, 30 Aug 2023 11:51:08 +0200 Subject: [PATCH 12/13] chore: added log msg to GpasPseudonymGenerator --- .../pseudonym/GpasPseudonymGenerator.java | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java b/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java index 91e465b..7b631d8 100644 --- a/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java +++ b/src/main/java/dev/dnpm/etl/processor/pseudonym/GpasPseudonymGenerator.java @@ -22,6 +22,21 @@ package dev.dnpm.etl.processor.pseudonym; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.parser.IParser; import dev.dnpm.etl.processor.config.GPasConfigProperties; +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.ConnectException; +import java.security.KeyManagementException; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.Base64; +import java.util.HashMap; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; import org.apache.commons.lang3.StringUtils; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; @@ -39,7 +54,11 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.http.*; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.retry.RetryCallback; import org.springframework.retry.RetryContext; @@ -51,22 +70,6 @@ import org.springframework.retry.support.RetryTemplate; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManagerFactory; -import java.io.BufferedInputStream; -import java.io.FileInputStream; -import java.io.IOException; -import java.net.ConnectException; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.security.cert.CertificateFactory; -import java.security.cert.X509Certificate; -import java.util.Base64; -import java.util.HashMap; - public class GpasPseudonymGenerator implements Generator { private final static FhirContext r4Context = FhirContext.forR4(); @@ -88,12 +91,16 @@ public class GpasPseudonymGenerator implements Generator { try { if (StringUtils.isNotBlank(gpasCfg.getSslCaLocation())) { customSslContext = getSslContext(gpasCfg.getSslCaLocation()); + log.debug(String.format("%s has been initialized with SSL certificate %s", + this.getClass().getName(), gpasCfg.getSslCaLocation())); } } catch (IOException | KeyManagementException | KeyStoreException | CertificateException | NoSuchAlgorithmException e) { throw new RuntimeException(e); } + log.debug(String.format("%s has been initialized", this.getClass().getName())); + } @Override @@ -115,9 +122,9 @@ public class GpasPseudonymGenerator implements Generator { } final var identifier = (Identifier) parameters.get().getPart().stream() - .filter(a -> a.getName().equals("pseudonym")) - .findFirst() - .orElseGet(ParametersParameterComponent::new).getValue(); + .filter(a -> a.getName().equals("pseudonym")) + .findFirst() + .orElseGet(ParametersParameterComponent::new).getValue(); // pseudonym return identifier.getSystem() + "|" + identifier.getValue(); From 8fd587c2a3dc0141e24fa462acc8244ac8a2bba3 Mon Sep 17 00:00:00 2001 From: Jakub Lidke Date: Wed, 30 Aug 2023 11:54:44 +0200 Subject: [PATCH 13/13] chore: remove unused HealthCheck.java --- HealthCheck.java | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 HealthCheck.java diff --git a/HealthCheck.java b/HealthCheck.java deleted file mode 100644 index 0aac2a6..0000000 --- a/HealthCheck.java +++ /dev/null @@ -1,20 +0,0 @@ -import java.io.IOException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse.BodyHandlers; - -public class HealthCheck { - - public static void main(String[] args) throws InterruptedException, IOException { - var client = HttpClient.newHttpClient(); - var request = HttpRequest.newBuilder() - .uri(URI.create("http://localhost:8001/actuator/health")) - .header("accept", "application/json") - .build(); - var response = client.send(request, BodyHandlers.ofString()); - if (response.statusCode() != 200 || !response.body().contains("UP")) { - throw new RuntimeException("Healthcheck failed"); - } - } -} \ No newline at end of file