1
0
mirror of https://github.com/pcvolkmer/etl-processor.git synced 2025-07-01 06:02:54 +00:00

57 Commits

Author SHA1 Message Date
f47b0b7de4 build: bump version 2024-03-12 13:27:45 +01:00
d8ba6b67cb docs: change description of ID anonymization 2024-03-12 13:27:29 +01:00
40b89dd4f1 Merge pull request #60 from CCC-MF/issue_44
feat: salted re-hash IDs within MTB file except patient ID
2024-03-12 13:18:32 +01:00
e3aeee61de feat: salted re-hash IDs within MTB file except patient ID 2024-03-12 13:13:31 +01:00
07e59f9b02 docs: update README.md to mention later kafka record processing 2024-03-09 11:09:44 +01:00
f751d64220 Merge pull request #59 from CCC-MF/issue_58
feat: do not use episode id in kafka record key
2024-03-09 11:00:28 +01:00
299bd56d63 feat: do not use episode id in kafka record key 2024-03-09 10:58:03 +01:00
a0c4d1863f Merge pull request #57 from CCC-MF/issue_56
feat: use requestId from incoming Kafka Record Header
2024-03-08 15:44:39 +01:00
fc1901211d feat: use requestId from incoming Kafka Record Header 2024-03-08 15:42:04 +01:00
bed91439db docs: update etl image 2024-03-07 18:30:44 +01:00
a8e008000e Merge pull request #54 from CCC-MF/issue_53
Anzeige gPAS Verbindungsstatus
2024-03-06 10:54:44 +01:00
a9c771aa99 test: change tests to mock output connection 2024-03-06 10:50:35 +01:00
256d9d4ff0 chore: change wording 2024-03-06 10:08:23 +01:00
41b87835ca feat: add configuration for deprecated config property 2024-03-06 10:02:31 +01:00
3654962294 feat: initial implementation of gPAS connection check 2024-03-06 10:00:17 +01:00
9382da7101 refactor: do not use singleton like rest template object 2024-03-05 17:03:10 +01:00
67ab0ef2be build: next snapshot version 2024-03-05 16:44:17 +01:00
69d796dab4 Merge pull request #52 from CCC-MF/issue_51
Darstellung und Aufteilung der Konfigurationsseite verbessern
2024-03-05 10:27:27 +01:00
4bfe7dc698 style: layout and style changes for config page 2024-03-05 10:24:25 +01:00
0aec5e4479 style: fixed first column width 2024-03-04 17:03:41 +01:00
b1a83510a6 style: fix statistics chart layout 2024-03-04 16:33:52 +01:00
6806c4fd69 build: bump version 2024-03-04 13:21:07 +01:00
b2016df852 style: fix some style issued 2024-03-04 13:17:57 +01:00
b332f3c5ff Merge pull request #50 from CCC-MF/issue_49
Administrative Rechte für OIDC-Benutzer
2024-03-04 12:54:31 +01:00
9eb65788e1 style: change login/logout style 2024-03-04 12:50:07 +01:00
9392bcadc9 feat: add admin role assignment 2024-03-04 10:12:12 +01:00
a008641192 fix: remove maxSessionsPreventsLogin 2024-03-01 15:07:23 +01:00
5928d52237 Merge pull request #48 from CCC-MF/issue_36
Freigabe und Berechtigung für OIDC-Benutzer
2024-03-01 14:09:06 +01:00
1eb40b40c9 docs: add documentation for user roles 2024-03-01 14:02:50 +01:00
feb9f2430c feat: add config page for user role assignment 2024-03-01 13:51:06 +01:00
200c5338ea feat: add default new user role config option 2024-03-01 09:34:51 +01:00
5c15ad4518 feat: add user role database table and role-based permissions 2024-03-01 09:30:07 +01:00
0b6decf88d Merge pull request #47 from CCC-MF/issue_43
feat: add config option to deactivate duplication check
2024-03-01 07:33:16 +01:00
cfdf41d550 feat: add config option to deactivate duplication check 2024-03-01 07:27:58 +01:00
45c65d53ce docs: add information about Kafka input topics 2024-02-29 13:55:14 +01:00
4568f491f5 Merge pull request #46 from CCC-MF/issue_42_kafka
Dateneingang über Apache-Kafka als Alternative zu HTTP-Request
2024-02-29 13:15:57 +01:00
952ad8c0cf test: add test for incoming kafka message processing 2024-02-29 12:49:06 +01:00
3e45bf8494 feat: implement KafkaInputListener 2024-02-29 09:19:32 +01:00
46ddaf10f7 Merge pull request #45 from CCC-MF/issue_34
Verwendung einer applikationsweiten Retry-Konfiguration
2024-02-29 08:57:00 +01:00
408b121f26 test: add test for max retry attempts 2024-02-29 08:47:17 +01:00
61e5273158 feat: add max-retry-attempts config option 2024-02-29 08:29:26 +01:00
50b8f7bbd4 feat: use global RetryTemplate 2024-02-29 08:26:54 +01:00
25f286f73b chore: update spring boot to version 3.2.3 2024-02-25 12:28:34 +01:00
50a6d66718 feat: new kafka config due to kafka input 2024-02-19 17:06:02 +01:00
f5c80f6d81 Merge pull request #41 from CCC-MF/issue_39
feat: add cache-control headers for static resources
2024-02-19 09:08:19 +01:00
7659939d3c Merge pull request #40 from CCC-MF/issue_37
build: use JDK 21 and update gradle version
2024-02-19 09:02:40 +01:00
f58d4a76cf build: use JDK 21 within workflows 2024-02-19 08:59:42 +01:00
c2dd450579 feat: add cache-control headers for static resources 2024-02-19 08:54:38 +01:00
a1b62ad754 build: use JDK 21 and update gradle version 2024-02-19 08:53:21 +01:00
59d8744c84 refactor: move mtb file controller into package input 2024-02-17 14:58:24 +01:00
d2a6ec17ea docs: add ports to example docker compose file 2024-02-16 13:00:22 +01:00
550403cc9f build: bump snapshot version 2024-02-15 13:41:11 +01:00
d3a4500568 Merge pull request #35 from CCC-MF/issue_33
Deprecate usage of ...SSL_CA_LOCATION config param
2024-02-09 08:19:07 +01:00
2e4fee97a8 feat: Deprecate usage of ...SSL_CA_LOCATION config param 2024-02-09 08:15:01 +01:00
5355eee05c docs: mark gPAS ssl config param as deprecated 2024-02-08 10:05:06 +01:00
3e22000541 Merge pull request #32 from CCC-MF/issue_31
build: update used actions
2024-02-05 08:18:26 +01:00
8c319197d0 build: update used actions 2024-02-05 08:12:28 +01:00
47 changed files with 1510 additions and 331 deletions

View File

@ -1,4 +1,4 @@
name: "Run build and deploy"
name: 'Run build and deploy'
on:
release:
@ -8,17 +8,17 @@ jobs:
docker:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
java-version: '17'
java-version: '21'
distribution: 'temurin'
- name: Setup Gradle
uses: gradle/gradle-build-action@v2.4.2
uses: gradle/actions/setup-gradle@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v2

View File

@ -11,14 +11,14 @@ jobs:
tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
java-version: '17'
java-version: '21'
distribution: 'temurin'
- name: Setup Gradle
uses: gradle/gradle-build-action@v2.4.2
uses: gradle/actions/setup-gradle@v3
- name: Execute tests
run: ./gradlew test
@ -26,14 +26,14 @@ jobs:
integrationTests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
java-version: '17'
java-version: '21'
distribution: 'temurin'
- name: Setup Gradle
uses: gradle/gradle-build-action@v2.4.2
uses: gradle/actions/setup-gradle@v3
- name: Execute integration tests
run: ./gradlew integrationTest

View File

@ -15,6 +15,11 @@ Zudem ist eine minimalistische Weboberfläche integriert, die einen Einblick in
![Modell DNPM-ETL-Strecke](docs/etl.png)
### Duplikaterkennung
Die Erkennung von Duplikaten ist normalerweise immer aktiv, kann jedoch über den Konfigurationsparameter
`APP_DUPLICATION_DETECTION=false` deaktiviert werden.
### Datenübermittlung über HTTP/REST
Anfragen werden, wenn nicht als Duplikat behandelt, nach der Pseudonymisierung direkt an das bwHC-Backend gesendet.
@ -33,22 +38,21 @@ Siehe hierzu auch: https://github.com/CCC-MF/kafka-to-bwhc
Wenn eine URI zu einer gPAS-Instanz (Version >= 2023.1.0) angegeben ist, wird diese verwendet.
Ist diese nicht gesetzt. wird intern eine Anonymisierung der Patienten-ID vorgenommen.
* `APP_PSEUDONYMIZE_PREFIX`: Standortbezogenes Prefix - `UNKNOWN`, wenn nicht gesetzt
* `APP_PSEUDONYMIZE_PREFIX`: Standortbezogenes Präfix - `UNKNOWN`, wenn nicht gesetzt
* `APP_PSEUDONYMIZE_GENERATOR`: `BUILDIN` oder `GPAS` - `BUILDIN`, wenn nicht gesetzt
**Hinweise**:
* Der alte Konfigurationsparameter `APP_PSEUDONYMIZER` mit den Werten `GPAS` oder `BUILDIN` sollte nicht
mehr verwendet werden.
* Der alte Konfigurationsparameter `APP_PSEUDONYMIZER` mit den Werten `GPAS` oder `BUILDIN` sollte nicht mehr verwendet
werden.
* Die Pseudonymisierung erfolgt im ETL-Prozessor nur für die Patienten-ID.
Andere Referenz-IDs werden nicht anonymisiert.
Dies erfolgt bei Nutzung von **[onkostar-plugin-dnpmexport](https://github.com/CCC-MF/onkostar-plugin-dnpmexport)**
bereits im Plugin selbst.
Andere IDs werden mithilfe des standortbezogenen Präfixes (erneut) anonymisiert, um für den aktuellen Kontext nicht
vergleichbare IDs bereitzustellen.
#### Eingebaute Anonymisierung
Wurde keine oder die Verwendung der eingebauten Anonymisierung konfiguriert, so wird für die Patienten-ID der
entsprechende SHA-256-Hash gebildet und Base64-codiert - hier ohne endende "=" - zuzüglich des konfigurierten Prefixes
entsprechende SHA-256-Hash gebildet und Base64-codiert - hier ohne endende "=" - zuzüglich des konfigurierten Präfixes
als Patienten-Pseudonym verwendet.
#### Pseudonymisierung mit gPAS
@ -59,7 +63,10 @@ Wurde die Verwendung von gPAS konfiguriert, so sind weitere Angaben zu konfiguri
* `APP_PSEUDONYMIZE_GPAS_TARGET`: gPas Domänenname
* `APP_PSEUDONYMIZE_GPAS_USERNAME`: gPas Basic-Auth Benutzername
* `APP_PSEUDONYMIZE_GPAS_PASSWORD`: gPas Basic-Auth Passwort
* `APP_PSEUDONYMIZE_GPAS_SSLCALOCATION`: Root Zertifikat für gPas, falls es dediziert hinzugefügt werden muss.
* ~~`APP_PSEUDONYMIZE_GPAS_SSLCALOCATION`~~: **Veraltet** - Root Zertifikat für gPas, falls es dediziert hinzugefügt werden muss.
Der Konfigurationsparameter `APP_PSEUDONYMIZE_GPAS_SSLCALOCATION` sollte nicht mehr verwendet werden und wird in einer kommenden Version entfernt.
Stattdessen sollte das Root Zertifikat wie unter [_Integration eines eigenen Root CA Zertifikats_](#integration-eines-eigenen-root-ca-zertifikats) beschrieben eingebunden werden.
### Anmeldung mit einem Passwort
@ -69,7 +76,7 @@ einem erfolgreichen Login erreichbar sind.
* `APP_SECURITY_ADMIN_USER`: Muss angegeben werden zur Aktivierung der Zugriffsbeschränkung.
* `APP_SECURITY_ADMIN_PASSWORD`: Das Passwort für den Administrator (Empfohlen).
Ein Administrator-Passwort muss inklusive des Encoding-Prefixes vorliegen.
Ein Administrator-Passwort muss inklusive des Encoding-Präfixes vorliegen.
Hier Beispiele für das Beispielpasswort `very-secret`:
@ -106,6 +113,23 @@ Weitere Informationen zur Konfiguration des OIDC-Providers
sind [hier](https://docs.spring.io/spring-security/reference/servlet/oauth2/index.html#oauth2-client)
zu finden.
#### Rollenbasierte Berechtigungen
Wird OpenID Connect verwendet, gibt es eine rollenbasierte Berechtigungszuweisung.
Die Standardrolle für neue OIDC-Benutzer kann mit der Option `APP_SECURITY_DEFAULT_USER_ROLE` festgelegt werden.
Mögliche Werte sind `user` oder `guest`. Standardwert ist `user`.
Benutzer mit der Rolle "Gast" sehen nur die Inhalte, die auch nicht angemeldete Benutzer sehen.
Hierdurch ist es möglich, einzelne Benutzer einzuschränken oder durch Änderung der Standardrolle auf `guest` nur
einzelne Benutzer als vollwertige Nutzer zuzulassen.
![Rollenverwaltung](docs/userroles.png)
Benutzer werden nach dem Entfernen oder der Änderung der vergebenen Rolle automatisch abgemeldet und müssen sich neu anmelden.
Sie bekommen dabei wieder die Standardrolle zugewiesen.
#### Auswirkungen auf den dargestellten Inhalt
Nur Administratoren haben Zugriff auf den Konfigurationsbereich, nur angemeldete Benutzer können die anonymisierte oder
@ -164,8 +188,10 @@ Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an das
Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an ein Kafka-Topic übermittelt wird:
* `APP_KAFKA_TOPIC`: Zu verwendendes Topic zum Versenden von Anfragen
* `APP_KAFKA_RESPONSE_TOPIC`: Topic mit Antworten über den Erfolg des Versendens. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_response".
* `APP_KAFKA_OUTPUT_TOPIC`: Zu verwendendes Topic zum Versenden von Anfragen.
Ersetzt in einer kommenden Version `APP_KAFKA_TOPIC`.
* `APP_KAFKA_OUTPUT_RESPONSE_TOPIC`: Topic mit Antworten über den Erfolg des Versendens. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_response".
Ersetzt in einer kommenden Version `APP_KAFKA_RESPONSE_TOPIC`.
* `APP_KAFKA_GROUP_ID`: Kafka GroupID des Consumers. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_group".
* `APP_KAFKA_SERVERS`: Zu verwendende Kafka-Bootstrap-Server als kommagetrennte Liste
@ -176,6 +202,8 @@ Weitere Einstellungen können über die Parameter von Spring Kafka konfiguriert
Lässt sich keine Verbindung zu dem bwHC-Backend aufbauen, wird eine Rückantwort mit Status-Code `900` erwartet, welchen es
für HTTP nicht gibt.
Wird die Umgebungsvariable `APP_KAFKA_INPUT_TOPIC` gesetzt, kann eine Nachricht auch über dieses Kafka-Topic an den ETL-Prozessor übermittelt werden.
##### Retention Time
Generell werden in Apache Kafka alle Records entsprechend der Konfiguration vorgehalten.
@ -205,16 +233,18 @@ kafka-configs.sh --bootstrap-server localhost:9092 --alter --topic test --add-co
kafka-configs.sh --bootstrap-server localhost:9092 --alter --topic test --add-config cleanup.policy=[delete,compact]
```
Da als Key eines Records die (pseudonymisierte) Patienten-ID und die (anonymisierte) Erkrankungs-ID verwendet wird,
stehen mit obiger Konfiguration der Kafka-Topics nach 10 Sekunden nur noch der jeweils letzte Eintrag für den entsprechenden
Key zur Verfügung.
Da als Key eines Records die (pseudonymisierte) Patienten-ID verwendet wird, stehen mit obiger Konfiguration
der Kafka-Topics nach 10 Sekunden nur noch der jeweils letzte Eintrag für den entsprechenden Key zur Verfügung.
Da der Key sowohl für die Records in Richtung bwHC-Backend für die Rückantwort identisch aufgebaut ist, lassen sich so
auch im Falle eines Consent-Widerspruchs die enthaltenen Daten als auch die Offenlegung durch Verifikationsdaten in der
Antwort effektiv verhindern, da diese nach 10 Sekunden gelöscht werden.
Es steht dann nur noch die jeweils letzten Information zur Verfügung, dass für einen Patienten/eine Erkrankung
ein Consent-Widerspruch erfolgte.
Dieses Vorgehen empfiehlt sich, wenn Sie gespeicherte Records nachgelagert für andere Auswertungen verwenden möchten.
## Docker-Images
Diese Anwendung ist auch als Docker-Image verfügbar: https://github.com/CCC-MF/etl-processor/pkgs/container/etl-processor
@ -276,7 +306,7 @@ auf Docker-Compose mit der gestartet werden kann.
Die Anwendung verarbeitet `X-Forwarded`-HTTP-Header und kann daher auch hinter einem Reverse-Proxy betrieben werden.
Dabei werden, je nachdem welche Header durch den Reverse-Proxy gesendet werden auch Protokoll, Host oder auch Path-Prefix
Dabei werden, je nachdem welche Header durch den Reverse-Proxy gesendet werden auch Protokoll, Host oder auch Path-Präfix
automatisch erkannt und verwendet werden. Dadurch ist z.B. eine abweichende Angabe des Pfads problemlos möglich.
#### Beispiel *Traefik* (mit Docker-Labels):

View File

@ -4,14 +4,14 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
plugins {
war
id("org.springframework.boot") version "3.2.2"
id("org.springframework.boot") version "3.2.3"
id("io.spring.dependency-management") version "1.1.4"
kotlin("jvm") version "1.9.22"
kotlin("plugin.spring") version "1.9.22"
}
group = "de.ukw.ccc"
version = "0.7.0"
version = "0.9.0"
var versions = mapOf(
"bwhc-dto-java" to "0.2.0",
@ -24,7 +24,7 @@ var versions = mapOf(
)
java {
sourceCompatibility = JavaVersion.VERSION_17
sourceCompatibility = JavaVersion.VERSION_21
}
sourceSets {
@ -90,7 +90,7 @@ dependencies {
tasks.withType<KotlinCompile> {
kotlinOptions {
freeCompilerArgs += "-Xjsr305=strict"
jvmTarget = "17"
jvmTarget = "21"
}
}

View File

@ -9,6 +9,8 @@ services:
### ETL-Processor
etl-processor:
image: ghcr.io/ccc-mf/etl-processor:latest
ports:
- 8080:8080
environment:
APP_REST_URI: http://bwhc-backend/bwhc/etl/api
SPRING_DATASOURCE_URL: jdbc:postgresql://postgres/etl

Binary file not shown.

Before

Width:  |  Height:  |  Size: 75 KiB

After

Width:  |  Height:  |  Size: 120 KiB

BIN
docs/userroles.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

View File

@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip
networkTimeout=10000
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

View File

@ -20,11 +20,13 @@
package dev.dnpm.etl.processor.config
import com.fasterxml.jackson.databind.ObjectMapper
import dev.dnpm.etl.processor.input.KafkaInputListener
import dev.dnpm.etl.processor.monitoring.RequestRepository
import dev.dnpm.etl.processor.output.KafkaMtbFileSender
import dev.dnpm.etl.processor.output.RestMtbFileSender
import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator
import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.dnpm.etl.processor.services.TokenRepository
import dev.dnpm.etl.processor.services.TokenService
import org.assertj.core.api.Assertions.assertThat
@ -37,6 +39,7 @@ import org.springframework.boot.test.context.SpringBootTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.boot.test.mock.mockito.MockBeans
import org.springframework.context.ApplicationContext
import org.springframework.retry.support.RetryTemplate
import org.springframework.security.crypto.password.PasswordEncoder
import org.springframework.security.provisioning.InMemoryUserDetailsManager
import org.springframework.test.context.ContextConfiguration
@ -78,8 +81,8 @@ class AppConfigurationTest {
@TestPropertySource(
properties = [
"app.kafka.servers=localhost:9092",
"app.kafka.topic=test",
"app.kafka.response-topic=test-response",
"app.kafka.output-topic=test",
"app.kafka.output-response-topic=test-response",
"app.kafka.group-id=test"
]
)
@ -99,8 +102,8 @@ class AppConfigurationTest {
properties = [
"app.rest.uri=http://localhost:9000",
"app.kafka.servers=localhost:9092",
"app.kafka.topic=test",
"app.kafka.response-topic=test-response",
"app.kafka.output-topic=test",
"app.kafka.output-response-topic=test-response",
"app.kafka.group-id=test"
]
)
@ -114,6 +117,44 @@ class AppConfigurationTest {
}
@Nested
@TestPropertySource(
properties = [
"app.kafka.servers=localhost:9092",
"app.kafka.output-topic=test",
"app.kafka.output-response-topic=test-response",
"app.kafka.group-id=test"
]
)
inner class AppConfigurationWithoutKafkaInputTest(private val context: ApplicationContext) {
@Test
fun shouldNotUseKafkaInputListener() {
assertThrows<NoSuchBeanDefinitionException> { context.getBean(KafkaInputListener::class.java) }
}
}
@Nested
@TestPropertySource(
properties = [
"app.kafka.servers=localhost:9092",
"app.kafka.input-topic=test_input",
"app.kafka.output-topic=test",
"app.kafka.output-response-topic=test-response",
"app.kafka.group-id=test"
]
)
@MockBean(RequestProcessor::class)
inner class AppConfigurationUsingKafkaInputTest(private val context: ApplicationContext) {
@Test
fun shouldUseKafkaInputListener() {
assertThat(context.getBean(KafkaInputListener::class.java)).isNotNull
}
}
@Nested
@TestPropertySource(
properties = [
@ -238,4 +279,30 @@ class AppConfigurationTest {
}
@Nested
@TestPropertySource(
properties = [
"app.rest.uri=http://localhost:9000",
"app.max-retry-attempts=5"
]
)
inner class AppConfigurationRetryTest(private val context: ApplicationContext) {
private val maxRetryAttempts = 5
@Test
fun shouldUseRetryTemplateWithConfiguredMaxAttempts() {
val retryTemplate = context.getBean(RetryTemplate::class.java)
assertThat(retryTemplate).isNotNull
assertThrows<RuntimeException> {
retryTemplate.execute<Void, RuntimeException> {
assertThat(it.retryCount).isLessThan(maxRetryAttempts)
throw RuntimeException()
}
}
}
}
}

View File

@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.web
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*

View File

@ -19,11 +19,14 @@
package dev.dnpm.etl.processor.web
import dev.dnpm.etl.processor.config.AppConfiguration
import dev.dnpm.etl.processor.config.AppSecurityConfiguration
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.RestConnectionCheckService
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.pseudonym.Generator
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.dnpm.etl.processor.services.TokenRepository
import dev.dnpm.etl.processor.services.TransformationService
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
@ -50,6 +53,7 @@ abstract class MockSink : Sinks.Many<Boolean>
@ContextConfiguration(
classes = [
ConfigController::class,
AppConfiguration::class,
AppSecurityConfiguration::class
]
)
@ -67,7 +71,9 @@ abstract class MockSink : Sinks.Many<Boolean>
MtbFileSender::class,
ConnectionCheckService::class,
RequestProcessor::class,
TransformationService::class
TransformationService::class,
TokenRepository::class,
RestConnectionCheckService::class
)
class ConfigControllerTest {

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -22,21 +22,6 @@ package dev.dnpm.etl.processor.pseudonym;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.parser.IParser;
import dev.dnpm.etl.processor.config.GPasConfigProperties;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.ConnectException;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.util.Base64;
import java.util.HashMap;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
@ -54,35 +39,41 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.*;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.RetryContext;
import org.springframework.retry.RetryListener;
import org.springframework.retry.RetryPolicy;
import org.springframework.retry.backoff.ExponentialBackOffPolicy;
import org.springframework.retry.policy.SimpleRetryPolicy;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.util.Base64;
public class GpasPseudonymGenerator implements Generator {
private final static FhirContext r4Context = FhirContext.forR4();
private final String gPasUrl;
private final String psnTargetDomain;
private final HttpHeaders httpHeader;
private final RetryTemplate retryTemplate = defaultTemplate();
private final RetryTemplate retryTemplate;
private final Logger log = LoggerFactory.getLogger(GpasPseudonymGenerator.class);
private SSLContext customSslContext;
private RestTemplate restTemplate;
private final RestTemplate restTemplate;
public GpasPseudonymGenerator(GPasConfigProperties gpasCfg) {
private SSLContext customSslContext;
public GpasPseudonymGenerator(GPasConfigProperties gpasCfg, RetryTemplate retryTemplate) {
this.retryTemplate = retryTemplate;
this.restTemplate = getRestTemplete();
this.gPasUrl = gpasCfg.getUri();
this.psnTargetDomain = gpasCfg.getTarget();
@ -91,7 +82,7 @@ public class GpasPseudonymGenerator implements Generator {
try {
if (StringUtils.isNotBlank(gpasCfg.getSslCaLocation())) {
customSslContext = getSslContext(gpasCfg.getSslCaLocation());
log.debug(String.format("%s has been initialized with SSL certificate %s",
log.warn(String.format("%s has been initialized with SSL certificate %s. This is deprecated in favor of including Root CA.",
this.getClass().getName(), gpasCfg.getSslCaLocation()));
}
} catch (IOException | KeyManagementException | KeyStoreException | CertificateException |
@ -150,7 +141,6 @@ public class GpasPseudonymGenerator implements Generator {
HttpEntity<String> requestEntity = new HttpEntity<>(gPasRequestBody, this.httpHeader);
ResponseEntity<String> responseEntity;
var restTemplate = getRestTemplete();
try {
responseEntity = retryTemplate.execute(
@ -202,31 +192,6 @@ public class GpasPseudonymGenerator implements Generator {
return headers;
}
protected RetryTemplate defaultTemplate() {
RetryTemplate retryTemplate = new RetryTemplate();
ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
backOffPolicy.setInitialInterval(1000);
backOffPolicy.setMultiplier(1.25);
retryTemplate.setBackOffPolicy(backOffPolicy);
HashMap<Class<? extends Throwable>, Boolean> retryableExceptions = new HashMap<>();
retryableExceptions.put(RestClientException.class, true);
retryableExceptions.put(ConnectException.class, true);
RetryPolicy retryPolicy = new SimpleRetryPolicy(3, retryableExceptions);
retryTemplate.setRetryPolicy(retryPolicy);
retryTemplate.registerListener(new RetryListener() {
@Override
public <T, E extends Throwable> void onError(RetryContext context,
RetryCallback<T, E> callback, Throwable throwable) {
log.warn("HTTP Error occurred: {}. Retrying {}", throwable.getMessage(),
context.getRetryCount());
RetryListener.super.onError(context, callback, throwable);
}
});
return retryTemplate;
}
/**
* Read SSL root certificate and return SSLContext
*
@ -262,14 +227,8 @@ public class GpasPseudonymGenerator implements Generator {
}
protected RestTemplate getRestTemplete() {
if (restTemplate != null) {
return restTemplate;
}
if (customSslContext == null) {
restTemplate = new RestTemplate();
return restTemplate;
return new RestTemplate();
}
final var sslsf = new SSLConnectionSocketFactory(customSslContext);
final Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
@ -282,7 +241,6 @@ public class GpasPseudonymGenerator implements Generator {
final HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(
httpClient);
restTemplate = new RestTemplate(requestFactory);
return restTemplate;
return new RestTemplate(requestFactory);
}
}

View File

@ -19,6 +19,7 @@
package dev.dnpm.etl.processor.config
import dev.dnpm.etl.processor.security.Role
import org.springframework.boot.context.properties.ConfigurationProperties
import org.springframework.boot.context.properties.DeprecatedConfigurationProperty
@ -30,7 +31,9 @@ data class AppConfigProperties(
replacement = "app.pseudonymize.generator"
)
var pseudonymizer: PseudonymGenerator = PseudonymGenerator.BUILDIN,
var transformations: List<TransformationProperties> = listOf()
var transformations: List<TransformationProperties> = listOf(),
var maxRetryAttempts: Int = 3,
var duplicationDetection: Boolean = true
) {
companion object {
const val NAME = "app"
@ -53,9 +56,11 @@ data class GPasConfigProperties(
val target: String = "etl-processor",
val username: String?,
val password: String?,
val sslCaLocation: String?,
) {
@get:DeprecatedConfigurationProperty(
reason = "Deprecated in favor of including Root CA"
)
val sslCaLocation: String?
) {
companion object {
const val NAME = "app.pseudonymize.gpas"
}
@ -70,10 +75,21 @@ data class RestTargetProperties(
}
}
@ConfigurationProperties(KafkaTargetProperties.NAME)
data class KafkaTargetProperties(
val topic: String = "etl-processor",
val responseTopic: String = "${topic}_response",
@ConfigurationProperties(KafkaProperties.NAME)
data class KafkaProperties(
val inputTopic: String?,
val outputTopic: String = "etl-processor",
@get:DeprecatedConfigurationProperty(
reason = "Deprecated",
replacement = "outputTopic"
)
val topic: String = outputTopic,
val outputResponseTopic: String = "${outputTopic}_response",
@get:DeprecatedConfigurationProperty(
reason = "Deprecated",
replacement = "outputResponseTopic"
)
val responseTopic: String = outputResponseTopic,
val groupId: String = "${topic}_group",
val servers: String = ""
) {
@ -87,7 +103,8 @@ data class SecurityConfigProperties(
val adminUser: String?,
val adminPassword: String?,
val enableTokens: Boolean = false,
val enableOidc: Boolean = false
val enableOidc: Boolean = false,
val defaultNewUserRole: Role = Role.USER
) {
companion object {
const val NAME = "app.security"

View File

@ -20,7 +20,7 @@
package dev.dnpm.etl.processor.config
import com.fasterxml.jackson.databind.ObjectMapper
import dev.dnpm.etl.processor.monitoring.ReportService
import dev.dnpm.etl.processor.monitoring.*
import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator
import dev.dnpm.etl.processor.pseudonym.Generator
import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator
@ -35,13 +35,16 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.retry.RetryCallback
import org.springframework.retry.RetryContext
import org.springframework.retry.RetryListener
import org.springframework.retry.policy.SimpleRetryPolicy
import org.springframework.retry.support.RetryTemplate
import org.springframework.retry.support.RetryTemplateBuilder
import org.springframework.scheduling.annotation.EnableScheduling
import org.springframework.security.crypto.password.PasswordEncoder
import org.springframework.security.provisioning.InMemoryUserDetailsManager
import org.springframework.security.provisioning.UserDetailsManager
import org.springframework.web.client.RestTemplate
import reactor.core.publisher.Sinks
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@ -60,10 +63,15 @@ class AppConfiguration {
private val logger = LoggerFactory.getLogger(AppConfiguration::class.java)
@Bean
fun restTemplate(): RestTemplate {
return RestTemplate()
}
@ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "GPAS")
@Bean
fun gpasPseudonymGenerator(configProperties: GPasConfigProperties): Generator {
return GpasPseudonymGenerator(configProperties)
fun gpasPseudonymGenerator(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate): Generator {
return GpasPseudonymGenerator(configProperties, retryTemplate)
}
@ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "BUILDIN", matchIfMissing = true)
@ -75,8 +83,8 @@ class AppConfiguration {
@ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS")
@ConditionalOnMissingBean
@Bean
fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties): Generator {
return GpasPseudonymGenerator(configProperties)
fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate): Generator {
return GpasPseudonymGenerator(configProperties, retryTemplate)
}
@ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "BUILDIN")
@ -111,11 +119,20 @@ class AppConfiguration {
}
@Bean
fun retryTemplate(): RetryTemplate {
fun retryTemplate(configProperties: AppConfigProperties): RetryTemplate {
return RetryTemplateBuilder()
.notRetryOn(IllegalArgumentException::class.java)
.fixedBackoff(5.seconds.toJavaDuration())
.customPolicy(SimpleRetryPolicy(3))
.exponentialBackoff(2.seconds.toJavaDuration(), 1.25, 5.seconds.toJavaDuration())
.customPolicy(SimpleRetryPolicy(configProperties.maxRetryAttempts))
.withListener(object : RetryListener {
override fun <T : Any, E : Throwable> onError(
context: RetryContext,
callback: RetryCallback<T, E>,
throwable: Throwable
) {
logger.warn("Error occured: {}. Retrying {}", throwable.message, context.retryCount)
}
})
.build()
}
@ -131,8 +148,29 @@ class AppConfiguration {
}
@Bean
fun configsUpdateProducer(): Sinks.Many<Boolean> {
return Sinks.many().multicast().directBestEffort()
fun connectionCheckUpdateProducer(): Sinks.Many<ConnectionCheckResult> {
return Sinks.many().multicast().onBackpressureBuffer()
}
@ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "GPAS")
@Bean
fun gPasConnectionCheckService(
restTemplate: RestTemplate,
gPasConfigProperties: GPasConfigProperties,
connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
): ConnectionCheckService {
return GPasConnectionCheckService(restTemplate, gPasConfigProperties, connectionCheckUpdateProducer)
}
@ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS")
@ConditionalOnMissingBean
@Bean
fun gPasConnectionCheckServiceOnDeprecatedProperty(
restTemplate: RestTemplate,
gPasConfigProperties: GPasConfigProperties,
connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
): ConnectionCheckService {
return GPasConnectionCheckService(restTemplate, gPasConfigProperties, connectionCheckUpdateProducer)
}
}

View File

@ -20,10 +20,13 @@
package dev.dnpm.etl.processor.config
import com.fasterxml.jackson.databind.ObjectMapper
import dev.dnpm.etl.processor.input.KafkaInputListener
import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.KafkaConnectionCheckService
import dev.dnpm.etl.processor.output.KafkaMtbFileSender
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.dnpm.etl.processor.services.kafka.KafkaResponseProcessor
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
@ -42,9 +45,9 @@ import reactor.core.publisher.Sinks
@Configuration
@EnableConfigurationProperties(
value = [KafkaTargetProperties::class]
value = [KafkaProperties::class]
)
@ConditionalOnProperty(value = ["app.kafka.topic", "app.kafka.servers"])
@ConditionalOnProperty(value = ["app.kafka.servers"])
@ConditionalOnMissingBean(MtbFileSender::class)
@Order(-5)
class AppKafkaConfiguration {
@ -54,21 +57,21 @@ class AppKafkaConfiguration {
@Bean
fun kafkaMtbFileSender(
kafkaTemplate: KafkaTemplate<String, String>,
kafkaTargetProperties: KafkaTargetProperties,
kafkaProperties: KafkaProperties,
retryTemplate: RetryTemplate,
objectMapper: ObjectMapper
): MtbFileSender {
logger.info("Selected 'KafkaMtbFileSender'")
return KafkaMtbFileSender(kafkaTemplate, kafkaTargetProperties, retryTemplate, objectMapper)
return KafkaMtbFileSender(kafkaTemplate, kafkaProperties, retryTemplate, objectMapper)
}
@Bean
fun kafkaListenerContainer(
fun kafkaResponseListenerContainer(
consumerFactory: ConsumerFactory<String, String>,
kafkaTargetProperties: KafkaTargetProperties,
kafkaProperties: KafkaProperties,
kafkaResponseProcessor: KafkaResponseProcessor
): KafkaMessageListenerContainer<String, String> {
val containerProperties = ContainerProperties(kafkaTargetProperties.responseTopic)
val containerProperties = ContainerProperties(kafkaProperties.responseTopic)
containerProperties.messageListener = kafkaResponseProcessor
return KafkaMessageListenerContainer(consumerFactory, containerProperties)
}
@ -82,8 +85,32 @@ class AppKafkaConfiguration {
}
@Bean
fun connectionCheckService(consumerFactory: ConsumerFactory<String, String>, configsUpdateProducer: Sinks.Many<Boolean>): ConnectionCheckService {
return KafkaConnectionCheckService(consumerFactory.createConsumer(), configsUpdateProducer)
@ConditionalOnProperty(value = ["app.kafka.input-topic"])
fun kafkaInputListenerContainer(
consumerFactory: ConsumerFactory<String, String>,
kafkaProperties: KafkaProperties,
kafkaInputListener: KafkaInputListener
): KafkaMessageListenerContainer<String, String> {
val containerProperties = ContainerProperties(kafkaProperties.inputTopic)
containerProperties.messageListener = kafkaInputListener
return KafkaMessageListenerContainer(consumerFactory, containerProperties)
}
@Bean
@ConditionalOnProperty(value = ["app.kafka.input-topic"])
fun kafkaInputListener(
requestProcessor: RequestProcessor,
objectMapper: ObjectMapper
): KafkaInputListener {
return KafkaInputListener(requestProcessor, objectMapper)
}
@Bean
fun kafkaConnectionCheckService(
consumerFactory: ConsumerFactory<String, String>,
connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
): ConnectionCheckService {
return KafkaConnectionCheckService(consumerFactory.createConsumer(), connectionCheckUpdateProducer)
}
}

View File

@ -19,6 +19,7 @@
package dev.dnpm.etl.processor.config
import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.RestConnectionCheckService
import dev.dnpm.etl.processor.output.MtbFileSender
@ -47,11 +48,6 @@ class AppRestConfiguration {
private val logger = LoggerFactory.getLogger(AppRestConfiguration::class.java)
@Bean
fun restTemplate(): RestTemplate {
return RestTemplate()
}
@Bean
fun restMtbFileSender(
restTemplate: RestTemplate,
@ -63,12 +59,12 @@ class AppRestConfiguration {
}
@Bean
fun connectionCheckService(
fun restConnectionCheckService(
restTemplate: RestTemplate,
restTargetProperties: RestTargetProperties,
configsUpdateProducer: Sinks.Many<Boolean>
connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
): ConnectionCheckService {
return RestConnectionCheckService(restTemplate, restTargetProperties, configsUpdateProducer)
return RestConnectionCheckService(restTemplate, restTargetProperties, connectionCheckUpdateProducer)
}
}

View File

@ -19,6 +19,9 @@
package dev.dnpm.etl.processor.config
import dev.dnpm.etl.processor.security.UserRole
import dev.dnpm.etl.processor.security.UserRoleRepository
import dev.dnpm.etl.processor.services.UserRoleService
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.boot.context.properties.EnableConfigurationProperties
@ -27,10 +30,15 @@ import org.springframework.context.annotation.Configuration
import org.springframework.security.config.annotation.web.builders.HttpSecurity
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity
import org.springframework.security.config.annotation.web.invoke
import org.springframework.security.core.authority.SimpleGrantedAuthority
import org.springframework.security.core.authority.mapping.GrantedAuthoritiesMapper
import org.springframework.security.core.session.SessionRegistry
import org.springframework.security.core.session.SessionRegistryImpl
import org.springframework.security.core.userdetails.User
import org.springframework.security.core.userdetails.UserDetails
import org.springframework.security.crypto.factory.PasswordEncoderFactories
import org.springframework.security.crypto.password.PasswordEncoder
import org.springframework.security.oauth2.core.oidc.user.OidcUserAuthority
import org.springframework.security.provisioning.InMemoryUserDetailsManager
import org.springframework.security.web.SecurityFilterChain
import java.util.*
@ -77,12 +85,19 @@ class AppSecurityConfiguration(
@Bean
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "true")
fun filterChainOidc(http: HttpSecurity, passwordEncoder: PasswordEncoder): SecurityFilterChain {
fun filterChainOidc(http: HttpSecurity, passwordEncoder: PasswordEncoder, userRoleRepository: UserRoleRepository, sessionRegistry: SessionRegistry): SecurityFilterChain {
http {
authorizeRequests {
authorize("/configs/**", hasRole("ADMIN"))
authorize("/mtbfile/**", hasAnyRole("MTBFILE"))
authorize("/report/**", fullyAuthenticated)
authorize("/report/**", hasAnyRole("ADMIN", "USER"))
authorize("*.css", permitAll)
authorize("*.ico", permitAll)
authorize("*.jpeg", permitAll)
authorize("*.js", permitAll)
authorize("*.svg", permitAll)
authorize("*.css", permitAll)
authorize("/login/**", permitAll)
authorize(anyRequest, permitAll)
}
httpBasic {
@ -94,11 +109,38 @@ class AppSecurityConfiguration(
oauth2Login {
loginPage = "/login"
}
sessionManagement {
sessionConcurrency {
maximumSessions = 1
expiredUrl = "/login?expired"
}
sessionFixation {
newSession()
}
}
csrf { disable() }
}
return http.build()
}
@Bean
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "true")
fun grantedAuthoritiesMapper(userRoleRepository: UserRoleRepository, appSecurityConfigProperties: SecurityConfigProperties): GrantedAuthoritiesMapper {
return GrantedAuthoritiesMapper { grantedAuthority ->
grantedAuthority.filterIsInstance<OidcUserAuthority>()
.onEach {
val userRole = userRoleRepository.findByUsername(it.userInfo.preferredUsername)
if (userRole.isEmpty) {
userRoleRepository.save(UserRole(null, it.userInfo.preferredUsername, appSecurityConfigProperties.defaultNewUserRole))
}
}
.map {
val userRole = userRoleRepository.findByUsername(it.userInfo.preferredUsername)
SimpleGrantedAuthority("ROLE_${userRole.get().role.toString().uppercase()}")
}
}
}
@Bean
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "false", matchIfMissing = true)
fun filterChain(http: HttpSecurity, passwordEncoder: PasswordEncoder): SecurityFilterChain {
@ -120,10 +162,19 @@ class AppSecurityConfiguration(
return http.build()
}
@Bean
fun sessionRegistry(): SessionRegistry {
return SessionRegistryImpl()
}
@Bean
fun passwordEncoder(): PasswordEncoder {
return PasswordEncoderFactories.createDelegatingPasswordEncoder()
}
@Bean
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "true")
fun userRoleService(userRoleRepository: UserRoleRepository, sessionRegistry: SessionRegistry): UserRoleService {
return UserRoleService(userRoleRepository, sessionRegistry)
}
}

View File

@ -0,0 +1,61 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.services.RequestProcessor
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.slf4j.LoggerFactory
import org.springframework.kafka.listener.MessageListener
class KafkaInputListener(
private val requestProcessor: RequestProcessor,
private val objectMapper: ObjectMapper
) : MessageListener<String, String> {
private val logger = LoggerFactory.getLogger(KafkaInputListener::class.java)
override fun onMessage(data: ConsumerRecord<String, String>) {
val mtbFile = objectMapper.readValue(data.value(), MtbFile::class.java)
val firstRequestIdHeader = data.headers().headers("requestId")?.firstOrNull()
val requestId = if (null != firstRequestIdHeader) {
String(firstRequestIdHeader.value())
} else {
""
}
if (mtbFile.consent.status == Consent.Status.ACTIVE) {
logger.debug("Accepted MTB File for processing")
if (requestId.isBlank()) {
requestProcessor.processMtbFile(mtbFile)
} else {
requestProcessor.processMtbFile(mtbFile, requestId)
}
} else {
logger.debug("Accepted MTB File and process deletion")
if (requestId.isBlank()) {
requestProcessor.processDeletion(mtbFile.patient.id)
} else {
requestProcessor.processDeletion(mtbFile.patient.id, requestId)
}
}
}
}

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.web
package dev.dnpm.etl.processor.input
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile

View File

@ -20,14 +20,21 @@
package dev.dnpm.etl.processor.monitoring
import dev.dnpm.etl.processor.config.GPasConfigProperties
import dev.dnpm.etl.processor.config.RestTargetProperties
import jakarta.annotation.PostConstruct
import org.apache.kafka.clients.consumer.Consumer
import org.apache.kafka.common.errors.TimeoutException
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.http.HttpEntity
import org.springframework.http.HttpHeaders
import org.springframework.http.HttpMethod
import org.springframework.http.HttpStatus
import org.springframework.http.MediaType
import org.springframework.http.RequestEntity
import org.springframework.scheduling.annotation.Scheduled
import org.springframework.web.client.RestTemplate
import org.springframework.web.util.UriComponentsBuilder
import reactor.core.publisher.Sinks
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@ -38,11 +45,22 @@ interface ConnectionCheckService {
}
interface OutputConnectionCheckService : ConnectionCheckService
sealed class ConnectionCheckResult {
abstract val available: Boolean
data class KafkaConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult()
data class RestConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult()
data class GPasConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult()
}
class KafkaConnectionCheckService(
private val consumer: Consumer<String, String>,
@Qualifier("configsUpdateProducer")
private val configsUpdateProducer: Sinks.Many<Boolean>
) : ConnectionCheckService {
@Qualifier("connectionCheckUpdateProducer")
private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
) : OutputConnectionCheckService {
private var connectionAvailable: Boolean = false
@ -55,7 +73,10 @@ class KafkaConnectionCheckService(
} catch (e: TimeoutException) {
false
}
configsUpdateProducer.emitNext(connectionAvailable, Sinks.EmitFailureHandler.FAIL_FAST)
connectionCheckUpdateProducer.emitNext(
ConnectionCheckResult.KafkaConnectionCheckResult(connectionAvailable),
Sinks.EmitFailureHandler.FAIL_FAST
)
}
override fun connectionAvailable(): Boolean {
@ -67,9 +88,9 @@ class KafkaConnectionCheckService(
class RestConnectionCheckService(
private val restTemplate: RestTemplate,
private val restTargetProperties: RestTargetProperties,
@Qualifier("configsUpdateProducer")
private val configsUpdateProducer: Sinks.Many<Boolean>
) : ConnectionCheckService {
@Qualifier("connectionCheckUpdateProducer")
private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
) : OutputConnectionCheckService {
private var connectionAvailable: Boolean = false
@ -84,7 +105,55 @@ class RestConnectionCheckService(
} catch (e: Exception) {
false
}
configsUpdateProducer.emitNext(connectionAvailable, Sinks.EmitFailureHandler.FAIL_FAST)
connectionCheckUpdateProducer.emitNext(
ConnectionCheckResult.RestConnectionCheckResult(connectionAvailable),
Sinks.EmitFailureHandler.FAIL_FAST
)
}
override fun connectionAvailable(): Boolean {
return this.connectionAvailable
}
}
class GPasConnectionCheckService(
private val restTemplate: RestTemplate,
private val gPasConfigProperties: GPasConfigProperties,
@Qualifier("connectionCheckUpdateProducer")
private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
) : ConnectionCheckService {
private var connectionAvailable: Boolean = false
@PostConstruct
@Scheduled(cron = "0 * * * * *")
fun check() {
connectionAvailable = try {
val uri = UriComponentsBuilder.fromUriString(
gPasConfigProperties.uri?.replace("/\$pseudonymizeAllowCreate", "/\$pseudonymize").toString()
)
.queryParam("target", gPasConfigProperties.target)
.queryParam("original", "???")
.build().toUri()
val headers = HttpHeaders()
headers.contentType = MediaType.APPLICATION_JSON
if (!gPasConfigProperties.username.isNullOrBlank() && !gPasConfigProperties.password.isNullOrBlank()) {
headers.setBasicAuth(gPasConfigProperties.username, gPasConfigProperties.password)
}
restTemplate.exchange(
uri,
HttpMethod.GET,
HttpEntity<Void>(headers),
Void::class.java
).statusCode == HttpStatus.OK
} catch (e: Exception) {
false
}
connectionCheckUpdateProducer.emitNext(
ConnectionCheckResult.GPasConnectionCheckResult(connectionAvailable),
Sinks.EmitFailureHandler.FAIL_FAST
)
}
override fun connectionAvailable(): Boolean {

View File

@ -22,7 +22,7 @@ package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.config.KafkaTargetProperties
import dev.dnpm.etl.processor.config.KafkaProperties
import dev.dnpm.etl.processor.monitoring.RequestStatus
import org.slf4j.LoggerFactory
import org.springframework.kafka.core.KafkaTemplate
@ -30,7 +30,7 @@ import org.springframework.retry.support.RetryTemplate
class KafkaMtbFileSender(
private val kafkaTemplate: KafkaTemplate<String, String>,
private val kafkaTargetProperties: KafkaTargetProperties,
private val kafkaProperties: KafkaProperties,
private val retryTemplate: RetryTemplate,
private val objectMapper: ObjectMapper
) : MtbFileSender {
@ -41,7 +41,7 @@ class KafkaMtbFileSender(
return try {
return retryTemplate.execute<MtbFileSender.Response, Exception> {
val result = kafkaTemplate.send(
kafkaTargetProperties.topic,
kafkaProperties.topic,
key(request),
objectMapper.writeValueAsString(Data(request.requestId, request.mtbFile))
)
@ -71,7 +71,7 @@ class KafkaMtbFileSender(
return try {
return retryTemplate.execute<MtbFileSender.Response, Exception> {
val result = kafkaTemplate.send(
kafkaTargetProperties.topic,
kafkaProperties.topic,
key(request),
objectMapper.writeValueAsString(Data(request.requestId, dummyMtbFile))
)
@ -90,12 +90,11 @@ class KafkaMtbFileSender(
}
override fun endpoint(): String {
return "${this.kafkaTargetProperties.servers} (${this.kafkaTargetProperties.topic}/${this.kafkaTargetProperties.responseTopic})"
return "${this.kafkaProperties.servers} (${this.kafkaProperties.topic}/${this.kafkaProperties.responseTopic})"
}
private fun key(request: MtbFileSender.MtbFileRequest): String {
return "{\"pid\": \"${request.mtbFile.patient.id}\", " +
"\"eid\": \"${request.mtbFile.episode.id}\"}"
return "{\"pid\": \"${request.mtbFile.patient.id}\"}"
}
private fun key(request: MtbFileSender.DeleteRequest): String {

View File

@ -33,4 +33,8 @@ class PseudonymizeService(
}
}
fun prefix(): String {
return configProperties.prefix
}
}

View File

@ -20,7 +20,14 @@
package dev.dnpm.etl.processor.pseudonym
import de.ukw.ccc.bwhc.dto.MtbFile
import org.apache.commons.codec.digest.DigestUtils
/** Replaces patient ID with generated patient pseudonym
*
* @param pseudonymizeService The pseudonymizeService to be used
*
* @return The MTB file containing patient pseudonymes
*/
infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
val patientPseudonym = pseudonymizeService.patientPseudonym(this.patient.id)
@ -46,8 +53,171 @@ infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
this.previousGuidelineTherapies.forEach { it.patient = patientPseudonym }
this.rebiopsyRequests.forEach { it.patient = patientPseudonym }
this.recommendations.forEach { it.patient = patientPseudonym }
this.recommendations.forEach { it.patient = patientPseudonym }
this.responses.forEach { it.patient = patientPseudonym }
this.studyInclusionRequests.forEach { it.patient = patientPseudonym }
this.specimens.forEach { it.patient = patientPseudonym }
}
/**
* Creates new hash of content IDs with given prefix except for patient IDs
*
* @param pseudonymizeService The pseudonymizeService to be used
*
* @return The MTB file containing rehashed content IDs
*/
infix fun MtbFile.anonymizeContentWith(pseudonymizeService: PseudonymizeService) {
val prefix = pseudonymizeService.prefix()
fun anonymize(id: String): String {
val hash = DigestUtils.sha256Hex("$prefix-$id").substring(0, 41).lowercase()
return "$prefix$hash"
}
this.episode.apply {
id = anonymize(id)
}
this.carePlans.onEach { carePlan ->
carePlan.apply {
id = anonymize(id)
diagnosis = anonymize(diagnosis)
geneticCounsellingRequest = anonymize(geneticCounsellingRequest)
rebiopsyRequests = rebiopsyRequests.map { anonymize(it) }
recommendations = recommendations.map { anonymize(it) }
studyInclusionRequests = studyInclusionRequests.map { anonymize(it) }
}
}
this.claims.onEach { claim ->
claim.apply {
id = anonymize(id)
therapy = anonymize(therapy)
}
}
this.claimResponses.onEach { claimResponse ->
claimResponse.apply {
id = anonymize(id)
claim = anonymize(claim)
}
}
this.consent.apply {
id = anonymize(id)
}
this.diagnoses.onEach { diagnosis ->
diagnosis.apply {
id = anonymize(id)
histologyResults = histologyResults.map { anonymize(it) }
}
}
this.ecogStatus.onEach { ecogStatus ->
ecogStatus.apply {
id = anonymize(id)
}
}
this.familyMemberDiagnoses.onEach { familyMemberDiagnosis ->
familyMemberDiagnosis.apply {
id = anonymize(id)
}
}
this.geneticCounsellingRequests.onEach { geneticCounsellingRequest ->
geneticCounsellingRequest.apply {
id = anonymize(id)
}
}
this.histologyReevaluationRequests.onEach { histologyReevaluationRequest ->
histologyReevaluationRequest.apply {
id = anonymize(id)
specimen = anonymize(specimen)
}
}
this.histologyReports.onEach { histologyReport ->
histologyReport.apply {
id = anonymize(id)
specimen = anonymize(specimen)
tumorMorphology.apply {
id = anonymize(id)
specimen = anonymize(specimen)
}
tumorCellContent.apply {
id = anonymize(id)
specimen = anonymize(specimen)
}
}
}
this.lastGuidelineTherapies.onEach { lastGuidelineTherapy ->
lastGuidelineTherapy.apply {
id = anonymize(id)
diagnosis = anonymize(diagnosis)
}
}
this.molecularPathologyFindings.onEach { molecularPathologyFinding ->
molecularPathologyFinding.apply {
id = anonymize(id)
specimen = anonymize(specimen)
}
}
this.molecularTherapies.onEach { molecularTherapy ->
molecularTherapy.apply {
history.onEach { history ->
history.apply {
id = anonymize(id)
basedOn = anonymize(basedOn)
}
}
}
}
this.ngsReports.onEach { ngsReport ->
ngsReport.apply {
id = anonymize(id)
specimen = anonymize(specimen)
tumorCellContent.apply {
id = anonymize(id)
specimen = anonymize(specimen)
}
simpleVariants.onEach { simpleVariant ->
simpleVariant.apply {
id = anonymize(id)
}
}
}
}
this.previousGuidelineTherapies.onEach { previousGuidelineTherapy ->
previousGuidelineTherapy.apply {
id = anonymize(id)
diagnosis = anonymize(diagnosis)
this.medication.forEach { medication ->
medication.apply {
id = anonymize(id)
}
}
}
}
this.rebiopsyRequests.onEach { rebiopsyRequest ->
rebiopsyRequest.apply {
id = anonymize(id)
specimen = anonymize(specimen)
}
}
this.recommendations.onEach { recommendation ->
recommendation.apply {
id = anonymize(id)
diagnosis = anonymize(diagnosis)
ngsReport = anonymize(ngsReport)
}
}
this.responses.onEach { response ->
response.apply {
id = anonymize(id)
therapy = anonymize(therapy)
}
}
this.studyInclusionRequests.onEach { studyInclusionRequest ->
studyInclusionRequest.apply {
id = anonymize(id)
reason = anonymize(reason)
}
}
this.specimens.onEach { specimen ->
specimen.apply {
id = anonymize(id)
}
}
}

View File

@ -0,0 +1,45 @@
/*
* This file is part of ETL-Processor
*
* Copyright (C) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package dev.dnpm.etl.processor.security
import org.springframework.data.annotation.Id
import org.springframework.data.relational.core.mapping.Table
import org.springframework.data.repository.CrudRepository
import java.util.*
@Table("user_role")
data class UserRole(
@Id val id: Long? = null,
val username: String,
var role: Role = Role.GUEST
)
enum class Role(val value: String) {
GUEST("guest"),
USER("user"),
ADMIN("admin")
}
interface UserRoleRepository : CrudRepository<UserRole, Long> {
fun findByUsername(username: String): Optional<UserRole>
}

View File

@ -21,12 +21,14 @@ package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.monitoring.Report
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
import dev.dnpm.etl.processor.pseudonym.anonymizeContentWith
import dev.dnpm.etl.processor.pseudonym.pseudonymizeWith
import org.apache.commons.codec.binary.Base32
import org.apache.commons.codec.digest.DigestUtils
@ -42,14 +44,19 @@ class RequestProcessor(
private val sender: MtbFileSender,
private val requestService: RequestService,
private val objectMapper: ObjectMapper,
private val applicationEventPublisher: ApplicationEventPublisher
private val applicationEventPublisher: ApplicationEventPublisher,
private val appConfigProperties: AppConfigProperties
) {
fun processMtbFile(mtbFile: MtbFile) {
val requestId = UUID.randomUUID().toString()
processMtbFile(mtbFile, UUID.randomUUID().toString())
}
fun processMtbFile(mtbFile: MtbFile, requestId: String) {
val pid = mtbFile.patient.id
mtbFile pseudonymizeWith pseudonymizeService
mtbFile anonymizeContentWith pseudonymizeService
val request = MtbFileSender.MtbFileRequest(requestId, transformationService.transform(mtbFile))
@ -64,7 +71,7 @@ class RequestProcessor(
)
)
if (isDuplication(mtbFile)) {
if (appConfigProperties.duplicationDetection && isDuplication(mtbFile)) {
applicationEventPublisher.publishEvent(
ResponseEvent(
requestId,
@ -101,8 +108,10 @@ class RequestProcessor(
}
fun processDeletion(patientId: String) {
val requestId = UUID.randomUUID().toString()
processDeletion(patientId, UUID.randomUUID().toString())
}
fun processDeletion(patientId: String, requestId: String) {
try {
val patientPseudonym = pseudonymizeService.patientPseudonym(patientId)

View File

@ -0,0 +1,61 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.services
import dev.dnpm.etl.processor.security.Role
import dev.dnpm.etl.processor.security.UserRole
import dev.dnpm.etl.processor.security.UserRoleRepository
import org.springframework.data.repository.findByIdOrNull
import org.springframework.security.core.session.SessionRegistry
import org.springframework.security.oauth2.core.oidc.user.OidcUser
class UserRoleService(
private val userRoleRepository: UserRoleRepository,
private val sessionRegistry: SessionRegistry
) {
fun updateUserRole(id: Long, role: Role) {
val userRole = userRoleRepository.findByIdOrNull(id) ?: return
userRole.role = role
userRoleRepository.save(userRole)
expireSessionFor(userRole.username)
}
fun deleteUserRole(id: Long) {
val userRole = userRoleRepository.findByIdOrNull(id) ?: return
userRoleRepository.delete(userRole)
expireSessionFor(userRole.username)
}
fun findAll(): List<UserRole> {
return userRoleRepository.findAll().toList()
}
private fun expireSessionFor(username: String) {
sessionRegistry.allPrincipals
.filterIsInstance<OidcUser>()
.filter { it.preferredUsername == username }
.flatMap {
sessionRegistry.getAllSessions(it, true)
}
.onEach {
it.expireNow()
}
}
}

View File

@ -19,12 +19,18 @@
package dev.dnpm.etl.processor.web
import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.GPasConnectionCheckService
import dev.dnpm.etl.processor.monitoring.OutputConnectionCheckService
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.pseudonym.Generator
import dev.dnpm.etl.processor.security.Role
import dev.dnpm.etl.processor.security.UserRole
import dev.dnpm.etl.processor.services.Token
import dev.dnpm.etl.processor.services.TokenService
import dev.dnpm.etl.processor.services.TransformationService
import dev.dnpm.etl.processor.services.UserRoleService
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.http.MediaType
import org.springframework.http.codec.ServerSentEvent
@ -37,37 +43,54 @@ import reactor.core.publisher.Sinks
@Controller
@RequestMapping(path = ["configs"])
class ConfigController(
@Qualifier("configsUpdateProducer")
private val configsUpdateProducer: Sinks.Many<Boolean>,
@Qualifier("connectionCheckUpdateProducer")
private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>,
private val transformationService: TransformationService,
private val pseudonymGenerator: Generator,
private val mtbFileSender: MtbFileSender,
private val connectionCheckService: ConnectionCheckService,
private val tokenService: TokenService?
private val connectionCheckServices: List<ConnectionCheckService>,
private val tokenService: TokenService?,
private val userRoleService: UserRoleService?
) {
@GetMapping
fun index(model: Model): String {
val outputConnectionAvailable =
connectionCheckServices.filterIsInstance<OutputConnectionCheckService>().first().connectionAvailable()
val gPasConnectionAvailable =
connectionCheckServices.filterIsInstance<GPasConnectionCheckService>().firstOrNull()?.connectionAvailable()
model.addAttribute("pseudonymGenerator", pseudonymGenerator.javaClass.simpleName)
model.addAttribute("mtbFileSender", mtbFileSender.javaClass.simpleName)
model.addAttribute("mtbFileEndpoint", mtbFileSender.endpoint())
model.addAttribute("connectionAvailable", connectionCheckService.connectionAvailable())
model.addAttribute("outputConnectionAvailable", outputConnectionAvailable)
model.addAttribute("gPasConnectionAvailable", gPasConnectionAvailable)
model.addAttribute("tokensEnabled", tokenService != null)
if (tokenService != null) {
model.addAttribute("tokens", tokenService.findAll())
} else {
model.addAttribute("tokens", listOf<Token>())
model.addAttribute("tokens", emptyList<Token>())
}
model.addAttribute("transformations", transformationService.getTransformations())
if (userRoleService != null) {
model.addAttribute("userRolesEnabled", true)
model.addAttribute("userRoles", userRoleService.findAll())
} else {
model.addAttribute("userRolesEnabled", false)
model.addAttribute("userRoles", emptyList<UserRole>())
}
return "configs"
}
@GetMapping(params = ["connectionAvailable"])
fun connectionAvailable(model: Model): String {
@GetMapping(params = ["outputConnectionAvailable"])
fun outputConnectionAvailable(model: Model): String {
val outputConnectionAvailable =
connectionCheckServices.filterIsInstance<OutputConnectionCheckService>().first().connectionAvailable()
model.addAttribute("mtbFileSender", mtbFileSender.javaClass.simpleName)
model.addAttribute("mtbFileEndpoint", mtbFileSender.endpoint())
model.addAttribute("connectionAvailable", connectionCheckService.connectionAvailable())
model.addAttribute("outputConnectionAvailable", outputConnectionAvailable)
if (tokenService != null) {
model.addAttribute("tokensEnabled", true)
model.addAttribute("tokens", tokenService.findAll())
@ -75,7 +98,25 @@ class ConfigController(
model.addAttribute("tokens", listOf<Token>())
}
return "configs/connectionAvailable"
return "configs/outputConnectionAvailable"
}
@GetMapping(params = ["gPasConnectionAvailable"])
fun gPasConnectionAvailable(model: Model): String {
val gPasConnectionAvailable =
connectionCheckServices.filterIsInstance<GPasConnectionCheckService>().firstOrNull()?.connectionAvailable()
model.addAttribute("mtbFileSender", mtbFileSender.javaClass.simpleName)
model.addAttribute("mtbFileEndpoint", mtbFileSender.endpoint())
model.addAttribute("gPasConnectionAvailable", gPasConnectionAvailable)
if (tokenService != null) {
model.addAttribute("tokensEnabled", true)
model.addAttribute("tokens", tokenService.findAll())
} else {
model.addAttribute("tokens", listOf<Token>())
}
return "configs/gPasConnectionAvailable"
}
@PostMapping(path = ["tokens"])
@ -112,11 +153,45 @@ class ConfigController(
return "configs/tokens"
}
@DeleteMapping(path = ["userroles/{id}"])
fun deleteUserRole(@PathVariable id: Long, model: Model): String {
if (userRoleService != null) {
userRoleService.deleteUserRole(id)
model.addAttribute("userRolesEnabled", true)
model.addAttribute("userRoles", userRoleService.findAll())
} else {
model.addAttribute("userRolesEnabled", false)
model.addAttribute("userRoles", emptyList<UserRole>())
}
return "configs/userroles"
}
@PutMapping(path = ["userroles/{id}"])
fun updateUserRole(@PathVariable id: Long, @ModelAttribute("role") role: Role, model: Model): String {
if (userRoleService != null) {
userRoleService.updateUserRole(id, role)
model.addAttribute("userRolesEnabled", true)
model.addAttribute("userRoles", userRoleService.findAll())
} else {
model.addAttribute("userRolesEnabled", false)
model.addAttribute("userRoles", emptyList<UserRole>())
}
return "configs/userroles"
}
@GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE])
fun events(): Flux<ServerSentEvent<Any>> {
return configsUpdateProducer.asFlux().map {
return connectionCheckUpdateProducer.asFlux().map {
val event = when (it) {
is ConnectionCheckResult.KafkaConnectionCheckResult -> "output-connection-check"
is ConnectionCheckResult.RestConnectionCheckResult -> "output-connection-check"
is ConnectionCheckResult.GPasConnectionCheckResult -> "gpas-connection-check"
}
ServerSentEvent.builder<Any>()
.event("connection-available").id("none").data("")
.event(event).id("none").data(it)
.build()
}
}

View File

@ -6,5 +6,16 @@ spring:
flyway:
locations: "classpath:db/migration/{vendor}"
web:
resources:
cache:
cachecontrol:
max-age: 1d
chain:
strategy:
content:
enabled: true
paths: /**/*.js,/**/*.css,/**/*.svg,/**/*.jpeg
server:
forward-headers-strategy: framework

View File

@ -0,0 +1,7 @@
CREATE TABLE IF NOT EXISTS user_role
(
id int auto_increment primary key,
username varchar(255) not null unique,
role varchar(255) not null,
created_at datetime default utc_timestamp() not null
);

View File

@ -0,0 +1,8 @@
CREATE TABLE IF NOT EXISTS user_role
(
id serial,
username varchar(255) not null unique,
role varchar(255) not null,
created_at timestamp with time zone default now() not null,
PRIMARY KEY (id)
);

View File

@ -2,6 +2,8 @@
--text: #333;
--table-border: rgba(16, 24, 40, .1);
--dark: brightness(.90);
--bg-blue: rgb(0, 74, 157);
--bg-blue-op: rgba(0, 74, 157, .35);
@ -40,7 +42,7 @@ body {
nav {
margin: 0 auto;
padding: 2em 0;
padding: 1em 0;
line-height: 1.5rem;
max-width: 1140px;
@ -48,18 +50,18 @@ nav {
border-bottom: 1px solid var(--table-border);
}
nav > a.nav-home {
nav a.nav-home {
float: left;
color: var(--text);
line-height: 1.5em;
text-decoration: none;
font-size: 1.5em;
font-size: 2em;
font-weight: bold;
}
nav > a.nav-home > img {
nav a.nav-home > img {
width: 1.5em;
vertical-align: middle;
}
@ -80,6 +82,57 @@ nav > ul > li.login {
margin: 0 0 0 1em;
padding: 0 0 0 2em;
border-left: 1px solid var(--table-border);
line-height: 3.5em;
}
nav > ul > li.login a {
text-decoration: none;
text-transform: none;
padding: 1em;
}
nav .login .user-name {
font-weight: bold;
}
nav > ul > li.login > span {
display: inline-block;
margin: 0 .5em;
}
nav > ul > li.login .user-icon {
flex-direction: column;
display: inline flex;
vertical-align: middle;
inline-size: 4em;
}
nav > ul > li.login .user-icon img {
margin: 0 0 -1em 0;
width: 80%;
align-self: center;
}
nav > ul > li.login .user-icon span {
padding: 0 .6em;
color: white;
font-size: .8em;
font-weight: bold;
border-radius: 4px;
line-height: normal;
text-align: center;
}
nav > ul > li.login .user-icon span.guest {
background: darkslategray;
}
nav > ul > li.login .user-icon span.user {
background: darkgreen;
}
nav > ul > li.login .user-icon span.admin {
background: darkred;
}
nav li a {
@ -89,10 +142,6 @@ nav li a {
font-weight: 700;
}
nav li.login a {
color: var(--bg-red);
}
nav li a:hover {
text-decoration: underline;
}
@ -202,6 +251,27 @@ form.samplecode-input input:focus-visible {
background: none;
}
.login-form img {
margin: 0 auto;
width: 4em;
display: block;
}
.userrole-form {
display: inline-block;
}
.userrole-form form {
margin: 0;
padding: 0;
border: none;
border-radius: 0;
background: none;
text-align: inherit;
}
.login-form form *,
.token-form form * {
padding: 0.5em;
@ -210,7 +280,8 @@ form.samplecode-input input:focus-visible {
}
.login-form form hr,
.token-form form hr {
.token-form form hr,
.userrole-form form hr {
padding: 0;
width: 100%;
}
@ -224,6 +295,14 @@ form.samplecode-input input:focus-visible {
border: none;
}
.userrole-form form select {
padding: 0.5em;
border: none;
border-radius: 3px;
line-height: 1.2rem;
font-size: 0.8rem;
}
.border {
padding: 1.5em;
border: 1px solid var(--table-border);
@ -246,6 +325,15 @@ table {
font-family: sans-serif;
}
table.config-table td:first-child {
width: 24em;
min-width: fit-content;
}
table.config-table td > button:last-of-type {
float: right;
}
.border > table {
padding: 0;
border: none;
@ -402,7 +490,6 @@ td.clipboard.clipped {
padding: 4px 8px;
line-height: 1.2rem;
vertical-align: middle;
border: 0 solid transparent;
border-radius: 3px;
@ -414,38 +501,38 @@ td.clipboard.clipped {
cursor: pointer;
}
.btn:active,
.btn:hover {
filter: drop-shadow(1px 2px 2px gray);
filter: drop-shadow(0px 1px 1px var(--bg-gray)) var(--dark);
}
.btn:active {
filter: drop-shadow(1px 1px 2px gray);
translate: 0 1px;
}
.btn.btn-red {
background: red;
background: var(--bg-red);
color: white;
}
.btn.btn-red:hover, .btn.btn-red:active {
background: darkred !important;
}
.btn.btn-blue {
background: slategray;
background: var(--bg-blue);
color: white;
}
.btn.btn-blue:hover, .btn.btn-blue:active {
background: darkslategray !important;
}
.btn.btn-delete:before {
content: '\1F5D1';
padding: .2rem;
}
button:disabled,
.btn:disabled {
background: slategray !important;
color: lightgray;
filter: none;
cursor: default;
}
input.inline {
border: none;
font-size: 1.1rem;
@ -481,15 +568,24 @@ input.inline:focus-visible {
font-weight: bold;
}
.chart {
width: calc(100% - 2.4rem - 4px);
height: 320px;
display: inline-block;
.charts {
display: grid;
grid-gap: 1em;
grid-template:
"a b" 28em
"c c" 28em / 1fr 1fr;
}
.chart-50pc {
width: calc(50% - 2.4rem - 4px);
.charts > .grid-left {
grid-area: a;
}
.charts > .grid-right {
grid-area: b;
}
.charts > .grid-full {
grid-area: c;
}
.connection-display {
@ -497,7 +593,7 @@ input.inline:focus-visible {
grid-template-columns: 10em 16em 10em;
place-items: center;
width: fit-content;
margin: 1em 0;
margin: 1em auto;
}
.connection-display > * {
@ -527,10 +623,40 @@ input.inline:focus-visible {
color: var(--bg-green);
}
.notification.notice {
color: var(--bg-yellow);
}
.notification.error {
color: var(--bg-red);
}
.tab {
padding: 1em;
border: none;
border-radius: 3px 3px 0 0;
cursor: pointer;
transition: all 0.2s;
font-weight: bold;
}
.tab:hover,
.tab.active {
background: var(--table-border);
}
.tabcontent {
border: 1px solid var(--table-border);
border-radius: 0 .5em .5em .5em;
display: none;
padding: 1em;
}
.tabcontent.active {
display: block;
}
a.reload {
display: none;
position: absolute;

View File

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg width="24" height="24" version="1.1" viewBox="0 0 6.35 6.35" xmlns="http://www.w3.org/2000/svg">
<g transform="matrix(1.2 0 0 1.2 -108.01 -85.977)">
<rect x="90.01" y="71.647" width="5.2917" height="5.2917" rx=".96212" fill="#b3b3b3"/>
<g transform="matrix(1.6667 0 0 1.6667 -60.888 -47.952)" fill="#fff">
<circle cx="92.126" cy="72.802" r=".70556"/>
<path d="m91.068 74.598a1.0583 1.0583 0 0 1 1.0583-1.0583 1.0583 1.0583 0 0 1 1.0583 1.0583h-1.0583z"/>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 588 B

View File

@ -10,87 +10,116 @@
<main>
<h1>Konfiguration</h1>
<section>
<h2>🔧 Allgemeine Konfiguration</h2>
<table>
<thead>
<tr>
<th>Name</th>
<th>Wert</th>
</tr>
</thead>
<tbody>
<div class="tabs">
<button class="tab active" onclick="selectTab(this, 'common');">Allgemeine Informationen</button>
<button class="tab" onclick="selectTab(this, 'security');">Sicherheit</button>
<button class="tab" onclick="selectTab(this, 'transformation');">Transformationen</button>
</div>
<div id="common" class="tabcontent active">
<section>
<h2>🔧 Allgemeine Konfiguration</h2>
<table class="config-table">
<thead>
<tr>
<td>Pseudonym erzeugt über</td>
<td>[[ ${pseudonymGenerator} ]]</td>
<th>Name</th>
<th>Wert</th>
</tr>
</thead>
<tbody>
<tr>
<td>Pseudonym erzeugt über</td>
<td>[[ ${pseudonymGenerator} ]]</td>
</tr>
<tr>
<td>MTBFile-Sender</td>
<td>[[ ${mtbFileSender} ]]</td>
</tr>
<tr>
<td th:if="${mtbFileSender.startsWith('Rest')}">REST-Endpunkt</td>
<td th:if="${mtbFileSender.startsWith('Kafka')}">Kafka-Broker und Topics</td>
<td>[[ ${mtbFileEndpoint} ]]</td>
</tr>
</tbody>
</table>
</section>
<section hx-ext="sse" th:sse-connect="@{/configs/events}">
<div th:insert="~{configs/gPasConnectionAvailable.html}" th:hx-get="@{/configs?gPasConnectionAvailable}" hx-trigger="sse:gpas-connection-check">
</div>
</section>
<section hx-ext="sse" th:sse-connect="@{/configs/events}">
<div th:insert="~{configs/outputConnectionAvailable.html}" th:hx-get="@{/configs?outputConnectionAvailable}" hx-trigger="sse:output-connection-check">
</div>
</section>
</div>
<div id="security" class="tabcontent">
<section th:insert="~{configs/tokens.html}">
</section>
<section th:insert="~{configs/userroles.html}">
</section>
</div>
<div id="transformation" class="tabcontent">
<section>
<h2><span th:if="${not transformations.isEmpty()}"></span><span th:if="${transformations.isEmpty()}"></span> Transformationen</h2>
<h3>Syntax</h3>
Hier einige Beispiele zum Syntax des JSON-Path
<ul>
<li style="padding: 0.6rem 0;"><span class="bg-path">diagnoses[*].icdO3T.version</span>: Ersetze die ICD-O3T-Version in allen Diagnosen, z.B. zur Version der deutschen Übersetzung</li>
<li style="padding: 0.6rem 0;"><span class="bg-path">patient.gender</span>: Ersetze das Geschlecht des Patienten, z.B. in das von bwHC verlangte Format</li>
</ul>
<h3>Konfigurierte Transformationen</h3>
<th:block th:if="${transformations.isEmpty()}">
<p>
Keine konfigurierten Transformationen.
</p>
</th:block>
<th:block th:if="${not transformations.isEmpty()}">
<p>
Hier sehen Sie eine Übersicht der konfigurierten Transformationen.
</p>
<table class="config-table">
<thead>
<tr>
<td>MTBFile-Sender</td>
<td>[[ ${mtbFileSender} ]]</td>
<th>JSON-Path</th>
<th>Transformation von &rArr; nach</th>
</tr>
<tr>
<td th:if="${mtbFileSender.startsWith('Rest')}">REST-Endpunkt</td>
<td th:if="${mtbFileSender.startsWith('Kafka')}">Kafka-Broker und Topics</td>
<td>[[ ${mtbFileEndpoint} ]]</td>
</thead>
<tbody>
<tr th:each="transformation : ${transformations}">
<td>
<span class="bg-path" title="Ersetze Wert(e) an dieser Stelle im MTB-File">[[ ${transformation.path} ]]</span>
</td>
<td>
<span class="bg-from" title="Ersetze immer dann, wenn dieser Wert enthalten ist">[[ ${transformation.existingValue} ]]</span>
<strong>&rArr;</strong>
<span class="bg-to" title="Ersetze durch diesen Wert">[[ ${transformation.newValue} ]]</span>
</td>
</tr>
</tbody>
</table>
</section>
<section th:insert="~{configs/tokens.html}">
</section>
<section hx-ext="sse" th:sse-connect="@{/configs/events}">
<div th:insert="~{configs/connectionAvailable.html}" th:hx-get="@{/configs?connectionAvailable}" hx-trigger="sse:connection-available">
</div>
</section>
<section>
<h2><span th:if="${not transformations.isEmpty()}"></span><span th:if="${transformations.isEmpty()}"></span> Transformationen</h2>
<h3>Syntax</h3>
Hier einige Beispiele zum Syntax des JSON-Path
<ul>
<li style="padding: 0.6rem 0;"><span class="bg-path">diagnoses[*].icdO3T.version</span>: Ersetze die ICD-O3T-Version in allen Diagnosen, z.B. zur Version der deutschen Übersetzung</li>
<li style="padding: 0.6rem 0;"><span class="bg-path">patient.gender</span>: Ersetze das Geschlecht des Patienten, z.B. in das von bwHC verlangte Format</li>
</ul>
<h3>Konfigurierte Transformationen</h3>
<th:block th:if="${transformations.isEmpty()}">
<p>
Keine konfigurierten Transformationen.
</p>
</th:block>
<th:block th:if="${not transformations.isEmpty()}">
<p>
Hier sehen Sie eine Übersicht der konfigurierten Transformationen.
</p>
<table>
<thead>
<tr>
<th>JSON-Path</th>
<th>Transformation von &rArr; nach</th>
</tr>
</thead>
<tbody>
<tr th:each="transformation : ${transformations}">
<td>
<span class="bg-path" title="Ersetze Wert(e) an dieser Stelle im MTB-File">[[ ${transformation.path} ]]</span>
</td>
<td>
<span class="bg-from" title="Ersetze immer dann, wenn dieser Wert enthalten ist">[[ ${transformation.existingValue} ]]</span>
<strong>&rArr;</strong>
<span class="bg-to" title="Ersetze durch diesen Wert">[[ ${transformation.newValue} ]]</span>
</td>
</tr>
</tbody>
</table>
</th:block>
</section>
</tbody>
</table>
</th:block>
</section>
</div>
</main>
<script th:src="@{/scripts.js}"></script>
<script th:src="@{/webjars/htmx.org/dist/htmx.min.js}"></script>
<script th:src="@{/webjars/htmx.org/dist/ext/sse.js}"></script>
<script>
function selectTab(self, elem) {
Array.from(document.getElementsByClassName('tab')).forEach(e => e.className = 'tab');
self.className = 'tab active';
Array.from(document.getElementsByClassName('tabcontent')).forEach(e => e.className = 'tabcontent');
document.getElementById(elem).className = 'tabcontent active';
}
</script>
</body>
</html>

View File

@ -0,0 +1,19 @@
<th:block th:if="${gPasConnectionAvailable == null}">
<h2><span>🟦</span> gPAS nicht konfiguriert - Patienten-IDs werden intern anonymisiert</h2>
</th:block>
<th:block th:if="${gPasConnectionAvailable != null}">
<h2><span th:if="${gPasConnectionAvailable}"></span><span th:if="${not(gPasConnectionAvailable)}"></span> Verbindung zu gPAS</h2>
<div>
Die Verbindung ist aktuell
<strong th:if="${gPasConnectionAvailable}" style="color: green">verfügbar.</strong>
<strong th:if="${not(gPasConnectionAvailable)}" style="color: red">nicht verfügbar.</strong>
</div>
<div class="connection-display border">
<img th:src="@{/server.png}" alt="ETL-Processor" />
<span class="connection" th:classappend="${gPasConnectionAvailable ? 'available' : ''}"></span>
<img th:src="@{/server.png}" alt="gPAS" />
<span>ETL-Processor</span>
<span></span>
<span>gPAS</span>
</div>
</th:block>

View File

@ -1,12 +1,12 @@
<h2><span th:if="${connectionAvailable}"></span><span th:if="${not(connectionAvailable)}"></span> Verbindung zum bwHC-Backend</h2>
<h2><span th:if="${outputConnectionAvailable}"></span><span th:if="${not(outputConnectionAvailable)}"></span> MTB-File Verbindung</h2>
<div>
Verbindung über <code>[[ ${mtbFileSender} ]]</code>. Die Verbindung ist aktuell
<strong th:if="${connectionAvailable}" style="color: green">verfügbar.</strong>
<strong th:if="${not(connectionAvailable)}" style="color: red">nicht verfügbar.</strong>
<strong th:if="${outputConnectionAvailable}" style="color: green">verfügbar.</strong>
<strong th:if="${not(outputConnectionAvailable)}" style="color: red">nicht verfügbar.</strong>
</div>
<div class="connection-display border">
<img th:src="@{/server.png}" alt="ETL-Processor" />
<span class="connection" th:classappend="${connectionAvailable ? 'available' : ''}"></span>
<span class="connection" th:classappend="${outputConnectionAvailable ? 'available' : ''}"></span>
<img th:if="${mtbFileSender.startsWith('Rest')}" th:src="@{/server.png}" alt="bwHC-Backend" />
<img th:if="${mtbFileSender.startsWith('Kafka')}" th:src="@{/kafka.png}" alt="Kafka-Broker" />
<span>ETL-Processor</span>

View File

@ -7,19 +7,20 @@
<h2><span></span> Tokens</h2>
<div class="border">
<div th:if="${tokens.isEmpty()}">Noch keine Tokens vorhanden.</div>
<table th:if="${not tokens.isEmpty()}">
<table th:if="${not tokens.isEmpty()}" class="config-table">
<thead>
<tr>
<th>Name</th>
<th>Erstellt</th>
<th></th>
</tr>
</thead>
<tbody>
<tr th:each="token : ${tokens}">
<td>[[ ${token.name} ]]</td>
<td><time th:datetime="${token.createdAt}">[[ ${token.createdAt} ]]</time></td>
<td><button class="btn btn-red" th:hx-delete="@{/configs/tokens/{id}(id=${token.id})}" hx-target="#tokens">Löschen</button></td>
<td>
<time th:datetime="${token.createdAt}">[[ ${token.createdAt} ]]</time>
<button class="btn btn-red" th:hx-delete="@{/configs/tokens/{id}(id=${token.id})}" hx-target="#tokens">Löschen</button>
</td>
</tr>
</tbody>
</table>
@ -32,7 +33,7 @@
<div class="token-form">
<form th:hx-post="@{/configs/tokens}" hx-target="#tokens">
<input placeholder="Token-Name" name="name" required />
<button>Token Erstellen</button>
<button class="btn">Token Erstellen</button>
</form>
</div>
</div>

View File

@ -0,0 +1,37 @@
<div th:if="${not userRolesEnabled}">
<h2><span></span> Benutzerberechtigungen</h2>
<p>Die Verwendung von rollenbasierten Benutzerberechtigungen ist nicht aktiviert.</p>
</div>
<div id="userroles" th:if="${userRolesEnabled}">
<h2><span></span> Benutzerberechtigungen</h2>
<div class="border">
<div th:if="${userRoles.isEmpty()}">Noch keine Benutzerberechtigungen vorhanden.</div>
<table th:if="${not userRoles.isEmpty()}" class="config-table">
<thead>
<tr>
<th>Benutzername</th>
<th>Rolle</th>
</tr>
</thead>
<tbody>
<tr th:each="userRole : ${userRoles}">
<td>[[ ${userRole.username} ]]</td>
<td>
<div class="userrole-form">
<form th:hx-put="@{/configs/userroles/{id}(id=${userRole.id})}" hx-target="#userroles">
<select name="role" th:disabled="${#authorization.authentication.getName() == userRole.username}">
<option th:selected="${userRole.role.value == 'guest'}" value="GUEST">Gast</option>
<option th:selected="${userRole.role.value == 'user'}" value="USER">Benutzer</option>
<option th:selected="${userRole.role.value == 'admin'}" value="ADMIN">Administrator</option>
</select>
<button class="btn btn-blue" th:disabled="${#authorization.authentication.getName() == userRole.username}">Übernehmen</button>
</form>
</div>
<button class="btn btn-red" th:hx-delete="@{/configs/userroles/{id}(id=${userRole.id})}" hx-target="#userroles" th:disabled="${#authorization.authentication.getName() == userRole.username}">Löschen</button>
</td>
</tr>
</tbody>
</table>
</div>
</div>

View File

@ -7,10 +7,12 @@
<body>
<div th:fragment="nav">
<nav>
<a class="nav-home" th:href="@{/}">
<img th:src="@{/icon.svg}" alt="Icon" />
<span>ETL-Processor</span>
</a>
<span>
<a class="nav-home" th:href="@{/}">
<img th:src="@{/icon.svg}" alt="Icon" />
<span>ETL-Processor</span>
</a>
</span>
<ul>
<li><a th:href="@{/}">Übersicht</a></li>
<li><a th:href="@{/statistics}">Statistiken</a></li>
@ -18,15 +20,19 @@
<a th:href="@{/configs}">Konfiguration</a>
</li>
<li class="login" sec:authorize="not isAuthenticated()">
<a th:href="@{/login}">Login</a>
<a class="btn btn-blue" th:href="@{/login}">Login</a>
</li>
<li class="login" sec:authorize="isAuthenticated()">
<span>
<span>👤</span>
<span sec:authentication="name">?</span>
<div class="user-icon">
<img th:src="@{/user.svg}" alt="User-Image">
<span sec:authorize="hasRole('ADMIN')" class="user-role admin">Admin</span>
<span sec:authorize="hasRole('USER')" class="user-role user">User</span>
<span sec:authorize="hasRole('GUEST')" class="user-role guest">Guest</span>
</div>
<span class="user-name" sec:authentication="name">?</span>
</span>
&nbsp;
<a th:href="@{/logout}">Abmelden</a>
<a class="btn btn-red" th:href="@{/logout}">Abmelden</a>
</li>
</ul>
</nav>

View File

@ -53,17 +53,17 @@
<td th:style="${request.type.value == 'delete'} ? 'color: red;'"><small>[[ ${request.type} ]]</small></td>
<td th:if="not ${request.report}">[[ ${request.uuid} ]]</td>
<td th:if="${request.report}">
<th:block sec:authorize="not authenticated">[[ ${request.uuid} ]]</th:block>
<a th:href="@{/report/{id}(id=${request.uuid})}" sec:authorize="authenticated">[[ ${request.uuid} ]]</a>
<a th:href="@{/report/{id}(id=${request.uuid})}" sec:authorize="hasRole('USER') or hasRole('ADMIN')">[[ ${request.uuid} ]]</a>
<th:block sec:authorize="not (hasRole('USER') or hasRole('ADMIN'))">[[ ${request.uuid} ]]</th:block>
</td>
<td><time th:datetime="${request.processedAt}">[[ ${request.processedAt} ]]</time></td>
<td class="patient-id" th:if="${patientId != null}" sec:authorize="authenticated">
<td class="patient-id" th:if="${patientId != null}" sec:authorize="hasRole('USER') or hasRole('ADMIN')">
[[ ${request.patientId} ]]
</td>
<td class="patient-id" th:if="${patientId == null}" sec:authorize="authenticated">
<td class="patient-id" th:if="${patientId == null}" sec:authorize="hasRole('USER') or hasRole('ADMIN')">
<a th:href="@{/patient/{pid}(pid=${request.patientId})}">[[ ${request.patientId} ]]</a>
</td>
<td class="patient-id" sec:authorize="not authenticated">***</td>
<td class="patient-id" sec:authorize="not (hasRole('USER') or hasRole('ADMIN'))">***</td>
</tr>
</tbody>
</table>

View File

@ -9,13 +9,15 @@
<div th:replace="~{fragments.html :: nav}"></div>
<main>
<div class="login-form">
<img th:src="@{/user.svg}" alt="user-logo" />
<h2 class="centered">Anmelden</h2>
<div class="centered notification error" th:if="${param.error}">Anmeldung nicht erfolgreich</div>
<div class="centered notification notice" th:if="${param.expired}">Sitzung abgelaufen oder von einem Administrator beendet.</div>
<div class="centered notification success" th:if="${param.logout}">Sie haben sich abgemeldet</div>
<form method="post" th:action="@{/login}">
<input type="text" id="username" name="username" class="form-control" placeholder="Username" required="" autofocus="" />
<input type="password" id="password" name="password" class="form-control" placeholder="Password" required="" />
<button type="submit">Anmelden</button>
<button class="btn" type="submit">Anmelden</button>
<hr th:if="${not oidcLogins.isEmpty()}" />
<a th:each="oidcLogin : ${oidcLogins}" class="btn" th:href="@{/oauth2/authorization/{provider}(provider=${oidcLogin.component1()})}">OIDC Login - [[ ${oidcLogin.component2()} ]]</a>
</form>

View File

@ -18,11 +18,11 @@
<p>
Anfragen zur Aktualisierung von Patientendaten durch Übermittlung eines MTB-Files.
</p>
<div>
<div id="piechart1" class="chart chart-50pc"></div>
<div id="piechart2" class="chart chart-50pc"></div>
<div class="charts">
<div id="piechart1" class="chart grid-left"></div>
<div id="piechart2" class="chart grid-right"></div>
<div id="barchart" class="chart grid-full"></div>
</div>
<div id="barchart" class="chart"></div>
</section>
<section>
@ -30,11 +30,11 @@
<p>
Anfragen zur Löschung von Patientendaten, wenn kein Consent vorliegt.
</p>
<div>
<div id="piechartdel1" class="chart chart-50pc"></div>
<div id="piechartdel2" class="chart chart-50pc"></div>
<div class="charts">
<div id="piechartdel1" class="chart grid-left"></div>
<div id="piechartdel2" class="chart grid-right"></div>
<div id="barchartdel" class="chart grid-full"></div>
</div>
<div id="barchartdel" class="chart"></div>
</section>
</main>

View File

@ -0,0 +1,112 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.services.RequestProcessor
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.header.internals.RecordHeader
import org.apache.kafka.common.header.internals.RecordHeaders
import org.apache.kafka.common.record.TimestampType
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.ArgumentMatchers.anyString
import org.mockito.Mock
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.any
import org.mockito.kotlin.times
import org.mockito.kotlin.verify
import java.util.*
@ExtendWith(MockitoExtension::class)
class KafkaInputListenerTest {
private lateinit var requestProcessor: RequestProcessor
private lateinit var objectMapper: ObjectMapper
private lateinit var kafkaInputListener: KafkaInputListener
@BeforeEach
fun setup(
@Mock requestProcessor: RequestProcessor
) {
this.requestProcessor = requestProcessor
this.objectMapper = ObjectMapper()
this.kafkaInputListener = KafkaInputListener(requestProcessor, objectMapper)
}
@Test
fun shouldProcessMtbFileRequest() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.ACTIVE).build())
.build()
kafkaInputListener.onMessage(ConsumerRecord("testtopic", 0, 0, "", this.objectMapper.writeValueAsString(mtbFile)))
verify(requestProcessor, times(1)).processMtbFile(any())
}
@Test
fun shouldProcessDeleteRequest() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
.build()
kafkaInputListener.onMessage(ConsumerRecord("testtopic", 0, 0, "", this.objectMapper.writeValueAsString(mtbFile)))
verify(requestProcessor, times(1)).processDeletion(anyString())
}
@Test
fun shouldProcessMtbFileRequestWithExistingRequestId() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.ACTIVE).build())
.build()
val headers = RecordHeaders(listOf(RecordHeader("requestId", UUID.randomUUID().toString().toByteArray())))
kafkaInputListener.onMessage(
ConsumerRecord("testtopic", 0, 0, -1L, TimestampType.NO_TIMESTAMP_TYPE, -1, -1, "", this.objectMapper.writeValueAsString(mtbFile), headers, Optional.empty())
)
verify(requestProcessor, times(1)).processMtbFile(any(), anyString())
}
@Test
fun shouldProcessDeleteRequestWithExistingRequestId() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
.build()
val headers = RecordHeaders(listOf(RecordHeader("requestId", UUID.randomUUID().toString().toByteArray())))
kafkaInputListener.onMessage(
ConsumerRecord("testtopic", 0, 0, -1L, TimestampType.NO_TIMESTAMP_TYPE, -1, -1, "", this.objectMapper.writeValueAsString(mtbFile), headers, Optional.empty())
)
verify(requestProcessor, times(1)).processDeletion(anyString(), anyString())
}
}

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.web
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*

View File

@ -21,7 +21,7 @@ package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.config.KafkaTargetProperties
import dev.dnpm.etl.processor.config.KafkaProperties
import dev.dnpm.etl.processor.monitoring.RequestStatus
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
@ -53,13 +53,13 @@ class KafkaMtbFileSenderTest {
fun setup(
@Mock kafkaTemplate: KafkaTemplate<String, String>
) {
val kafkaTargetProperties = KafkaTargetProperties("testtopic")
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(1)).build()
this.objectMapper = ObjectMapper()
this.kafkaTemplate = kafkaTemplate
this.kafkaMtbFileSender = KafkaMtbFileSender(kafkaTemplate, kafkaTargetProperties, retryTemplate, objectMapper)
this.kafkaMtbFileSender = KafkaMtbFileSender(kafkaTemplate, kafkaProperties, retryTemplate, objectMapper)
}
@ParameterizedTest
@ -101,7 +101,7 @@ class KafkaMtbFileSenderTest {
val captor = argumentCaptor<String>()
verify(kafkaTemplate, times(1)).send(anyString(), captor.capture(), captor.capture())
assertThat(captor.firstValue).isNotNull
assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\", \"eid\": \"1\"}")
assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\"}")
assertThat(captor.secondValue).isNotNull
assertThat(captor.secondValue).isEqualTo(objectMapper.writeValueAsString(kafkaRecordData("TestID", Consent.Status.ACTIVE)))
}
@ -125,9 +125,9 @@ class KafkaMtbFileSenderTest {
@ParameterizedTest
@MethodSource("requestWithResponseSource")
fun shouldRetryOnMtbFileKafkaSendError(testData: TestData) {
val kafkaTargetProperties = KafkaTargetProperties("testtopic")
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(3)).build()
this.kafkaMtbFileSender = KafkaMtbFileSender(this.kafkaTemplate, kafkaTargetProperties, retryTemplate, this.objectMapper)
this.kafkaMtbFileSender = KafkaMtbFileSender(this.kafkaTemplate, kafkaProperties, retryTemplate, this.objectMapper)
doAnswer {
if (null != testData.exception) {
@ -151,9 +151,9 @@ class KafkaMtbFileSenderTest {
@ParameterizedTest
@MethodSource("requestWithResponseSource")
fun shouldRetryOnDeleteKafkaSendError(testData: TestData) {
val kafkaTargetProperties = KafkaTargetProperties("testtopic")
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(3)).build()
this.kafkaMtbFileSender = KafkaMtbFileSender(this.kafkaTemplate, kafkaTargetProperties, retryTemplate, this.objectMapper)
this.kafkaMtbFileSender = KafkaMtbFileSender(this.kafkaTemplate, kafkaProperties, retryTemplate, this.objectMapper)
doAnswer {
if (null != testData.exception) {

View File

@ -20,9 +20,10 @@
package dev.dnpm.etl.processor.pseudonym
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.MtbFile
import de.ukw.ccc.bwhc.dto.*
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.ArgumentMatchers
import org.mockito.Mock
@ -61,4 +62,76 @@ class ExtensionsTest {
assertThat(mtbFile.serialized()).doesNotContain(CLEAN_PATIENT_ID)
}
@Test
fun shouldNotContainAnyUuidAfterRehashingOfIds(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(ArgumentMatchers.anyString())
doAnswer {
"TESTDOMAIN"
}.whenever(pseudonymizeService).prefix()
val mtbFile = fakeMtbFile()
mtbFile.pseudonymizeWith(pseudonymizeService)
mtbFile.anonymizeContentWith(pseudonymizeService)
val pattern = "\"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\"".toRegex().toPattern()
val matcher = pattern.matcher(mtbFile.serialized())
assertThrows<IllegalStateException> {
matcher.find()
matcher.group()
}.also {
assertThat(it.message).isEqualTo("No match found")
}
}
@Test
fun shouldRehashIdsWithPrefix(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(ArgumentMatchers.anyString())
doAnswer {
"TESTDOMAIN"
}.whenever(pseudonymizeService).prefix()
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.build()
mtbFile.pseudonymizeWith(pseudonymizeService)
mtbFile.anonymizeContentWith(pseudonymizeService)
assertThat(mtbFile.episode.id)
// TESTDOMAIN<sha256(TESTDOMAIN-1)[0-41]>
.isEqualTo("TESTDOMAIN44e20a53bbbf9f3ae39626d05df7014dcd77d6098")
}
}

View File

@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
@ -51,6 +52,7 @@ class RequestProcessorTest {
private lateinit var sender: MtbFileSender
private lateinit var requestService: RequestService
private lateinit var applicationEventPublisher: ApplicationEventPublisher
private lateinit var appConfigProperties: AppConfigProperties
private lateinit var requestProcessor: RequestProcessor
@ -67,6 +69,7 @@ class RequestProcessorTest {
this.sender = sender
this.requestService = requestService
this.applicationEventPublisher = applicationEventPublisher
this.appConfigProperties = AppConfigProperties(null)
val objectMapper = ObjectMapper()
@ -76,7 +79,8 @@ class RequestProcessorTest {
sender,
requestService,
objectMapper,
applicationEventPublisher
applicationEventPublisher,
appConfigProperties
)
}
@ -88,7 +92,7 @@ class RequestProcessorTest {
uuid = UUID.randomUUID().toString(),
patientId = "TEST_12345678901",
pid = "P1",
fingerprint = "xrysxpozhbs2lnrjgf3yq4fzj33kxr7xr5c2cbuskmelfdmckl3a",
fingerprint = "zdlzv5s5ydmd4ktw2v5piohegc4jcyrm6j66bq6tv2uxuerndmga",
type = RequestType.MTB_FILE,
status = RequestStatus.SUCCESS,
processedAt = Instant.parse("2023-08-08T02:00:00Z")
@ -147,7 +151,7 @@ class RequestProcessorTest {
uuid = UUID.randomUUID().toString(),
patientId = "TEST_12345678901",
pid = "P1",
fingerprint = "xrysxpozhbs2lnrjgf3yq4fzj33kxr7xr5c2cbuskmelfdmckl3a",
fingerprint = "zdlzv5s5ydmd4ktw2v5piohegc4jcyrm6j66bq6tv2uxuerndmga",
type = RequestType.MTB_FILE,
status = RequestStatus.SUCCESS,
processedAt = Instant.parse("2023-08-08T02:00:00Z")
@ -390,4 +394,52 @@ class RequestProcessorTest {
assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR)
}
@Test
fun testShouldNotDetectMtbFileDuplicationIfDuplicationNotConfigured() {
this.appConfigProperties.duplicationDetection = false
doAnswer {
it.arguments[0] as String
}.`when`(pseudonymizeService).patientPseudonym(any())
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
doAnswer {
MtbFileSender.Response(status = RequestStatus.SUCCESS)
}.`when`(sender).send(any<MtbFileSender.MtbFileRequest>())
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.build()
this.requestProcessor.processMtbFile(mtbFile)
val eventCaptor = argumentCaptor<ResponseEvent>()
verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture())
assertThat(eventCaptor.firstValue).isNotNull
assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS)
}
}

File diff suppressed because one or more lines are too long