1
0
mirror of https://github.com/pcvolkmer/etl-processor.git synced 2025-07-01 14:12:55 +00:00

49 Commits

Author SHA1 Message Date
b45a0ba609 chore: bump version 2025-03-22 23:41:17 +01:00
d49671f0d4 build: update image name 2025-03-22 23:40:13 +01:00
3a19212a78 chore: update Spring Boot 2025-03-09 09:38:59 +01:00
e95fa2fb12 build: change BP_OCI_SOURCE 2024-11-01 15:06:24 +01:00
1bcc8c13de build: change group name 2024-11-01 14:44:09 +01:00
2fc3299543 build: replace hard coded repo name with variable (#81) 2024-11-01 14:23:26 +01:00
5575867632 chore: update to Spring Boot 3.2.11 (#80) 2024-11-01 14:14:45 +01:00
87658bfa58 chore: update to Spring Boot 3.2.7 2024-07-15 07:58:36 +02:00
99efd6c98a fix: downgrade echarts due to dependency issues 2024-07-15 07:43:44 +02:00
e42d11f125 chore: update webjars dependencies 2024-07-15 07:43:32 +02:00
8b194e7212 chore: update kotlin version 2024-05-31 12:55:32 +02:00
070100eba0 chore: update spring dependency-management plugin 2024-05-31 12:54:28 +02:00
ce1489d9a1 chore: update spring boot dependencies 2024-05-31 12:54:23 +02:00
ca1e73a0b5 chore: update bwhc-dto-java dependency 2024-05-31 12:54:18 +02:00
041bf459ef fix: add missing 'fatal' severity 2024-05-31 12:52:22 +02:00
c922e27758 fix: handle null values in MtbFile
This should not occur but if, it should not result in NPE except for

* Patient
* Consent
* Episode
2024-05-31 12:51:40 +02:00
4d5c0ce1fb chore: remove println 2024-05-31 12:51:36 +02:00
bb0bbf5a28 chore: update webjars-locator dependency 2024-05-31 12:49:51 +02:00
1b4585d601 docs: fix CVE number in dependency comment 2024-05-31 12:44:23 +02:00
dad3ea80ee chore: update integration test dependency
This mitigates CVE-204-26308 and CVE-2024-25710
2024-05-31 12:44:19 +02:00
01446bdece chore: update GitHub workflow actions 2024-05-31 12:34:57 +02:00
43660a4dcb chore: mark as snapshot version 2024-05-31 12:22:08 +02:00
8313420de5 chore: bump version for new release 2024-04-19 09:42:19 +02:00
1651f446fe chore: update spring boot and other dependencies 2024-04-19 09:41:17 +02:00
056a087065 chore: update spring boot dependencies 2024-03-25 16:12:20 +01:00
a730ce2a53 fix: update spring security due to CVE-2024-22257 2024-03-19 16:40:32 +01:00
12eb1feea6 fix: assign new value from scope function 2024-03-12 18:29:42 +01:00
af714f7b64 fix: ignore possible null values in mtb files 2024-03-12 17:56:25 +01:00
f47b0b7de4 build: bump version 2024-03-12 13:27:45 +01:00
d8ba6b67cb docs: change description of ID anonymization 2024-03-12 13:27:29 +01:00
40b89dd4f1 Merge pull request #60 from CCC-MF/issue_44
feat: salted re-hash IDs within MTB file except patient ID
2024-03-12 13:18:32 +01:00
e3aeee61de feat: salted re-hash IDs within MTB file except patient ID 2024-03-12 13:13:31 +01:00
07e59f9b02 docs: update README.md to mention later kafka record processing 2024-03-09 11:09:44 +01:00
f751d64220 Merge pull request #59 from CCC-MF/issue_58
feat: do not use episode id in kafka record key
2024-03-09 11:00:28 +01:00
299bd56d63 feat: do not use episode id in kafka record key 2024-03-09 10:58:03 +01:00
a0c4d1863f Merge pull request #57 from CCC-MF/issue_56
feat: use requestId from incoming Kafka Record Header
2024-03-08 15:44:39 +01:00
fc1901211d feat: use requestId from incoming Kafka Record Header 2024-03-08 15:42:04 +01:00
bed91439db docs: update etl image 2024-03-07 18:30:44 +01:00
a8e008000e Merge pull request #54 from CCC-MF/issue_53
Anzeige gPAS Verbindungsstatus
2024-03-06 10:54:44 +01:00
a9c771aa99 test: change tests to mock output connection 2024-03-06 10:50:35 +01:00
256d9d4ff0 chore: change wording 2024-03-06 10:08:23 +01:00
41b87835ca feat: add configuration for deprecated config property 2024-03-06 10:02:31 +01:00
3654962294 feat: initial implementation of gPAS connection check 2024-03-06 10:00:17 +01:00
9382da7101 refactor: do not use singleton like rest template object 2024-03-05 17:03:10 +01:00
67ab0ef2be build: next snapshot version 2024-03-05 16:44:17 +01:00
69d796dab4 Merge pull request #52 from CCC-MF/issue_51
Darstellung und Aufteilung der Konfigurationsseite verbessern
2024-03-05 10:27:27 +01:00
4bfe7dc698 style: layout and style changes for config page 2024-03-05 10:24:25 +01:00
0aec5e4479 style: fixed first column width 2024-03-04 17:03:41 +01:00
b1a83510a6 style: fix statistics chart layout 2024-03-04 16:33:52 +01:00
33 changed files with 819 additions and 224 deletions

View File

@ -21,7 +21,7 @@ jobs:
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -30,6 +30,6 @@ jobs:
- name: Execute image build and push
run: |
./gradlew bootBuildImage
docker tag ghcr.io/ccc-mf/etl-processor ghcr.io/ccc-mf/etl-processor:${{ github.ref_name }}
docker push ghcr.io/ccc-mf/etl-processor
docker push ghcr.io/ccc-mf/etl-processor:${{ github.ref_name }}
docker tag ghcr.io/${{ github.repository }} ghcr.io/${{ github.repository }}:${{ github.ref_name }}
docker push ghcr.io/${{ github.repository }}
docker push ghcr.io/${{ github.repository }}:${{ github.ref_name }}

View File

@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
- uses: actions/setup-java@v4
with:
java-version: '21'
distribution: 'temurin'
@ -27,7 +27,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
- uses: actions/setup-java@v4
with:
java-version: '21'
distribution: 'temurin'

View File

@ -38,22 +38,21 @@ Siehe hierzu auch: https://github.com/CCC-MF/kafka-to-bwhc
Wenn eine URI zu einer gPAS-Instanz (Version >= 2023.1.0) angegeben ist, wird diese verwendet.
Ist diese nicht gesetzt. wird intern eine Anonymisierung der Patienten-ID vorgenommen.
* `APP_PSEUDONYMIZE_PREFIX`: Standortbezogenes Prefix - `UNKNOWN`, wenn nicht gesetzt
* `APP_PSEUDONYMIZE_PREFIX`: Standortbezogenes Präfix - `UNKNOWN`, wenn nicht gesetzt
* `APP_PSEUDONYMIZE_GENERATOR`: `BUILDIN` oder `GPAS` - `BUILDIN`, wenn nicht gesetzt
**Hinweise**:
* Der alte Konfigurationsparameter `APP_PSEUDONYMIZER` mit den Werten `GPAS` oder `BUILDIN` sollte nicht
mehr verwendet werden.
* Der alte Konfigurationsparameter `APP_PSEUDONYMIZER` mit den Werten `GPAS` oder `BUILDIN` sollte nicht mehr verwendet
werden.
* Die Pseudonymisierung erfolgt im ETL-Prozessor nur für die Patienten-ID.
Andere Referenz-IDs werden nicht anonymisiert.
Dies erfolgt bei Nutzung von **[onkostar-plugin-dnpmexport](https://github.com/CCC-MF/onkostar-plugin-dnpmexport)**
bereits im Plugin selbst.
Andere IDs werden mithilfe des standortbezogenen Präfixes (erneut) anonymisiert, um für den aktuellen Kontext nicht
vergleichbare IDs bereitzustellen.
#### Eingebaute Anonymisierung
Wurde keine oder die Verwendung der eingebauten Anonymisierung konfiguriert, so wird für die Patienten-ID der
entsprechende SHA-256-Hash gebildet und Base64-codiert - hier ohne endende "=" - zuzüglich des konfigurierten Prefixes
entsprechende SHA-256-Hash gebildet und Base64-codiert - hier ohne endende "=" - zuzüglich des konfigurierten Präfixes
als Patienten-Pseudonym verwendet.
#### Pseudonymisierung mit gPAS
@ -77,7 +76,7 @@ einem erfolgreichen Login erreichbar sind.
* `APP_SECURITY_ADMIN_USER`: Muss angegeben werden zur Aktivierung der Zugriffsbeschränkung.
* `APP_SECURITY_ADMIN_PASSWORD`: Das Passwort für den Administrator (Empfohlen).
Ein Administrator-Passwort muss inklusive des Encoding-Prefixes vorliegen.
Ein Administrator-Passwort muss inklusive des Encoding-Präfixes vorliegen.
Hier Beispiele für das Beispielpasswort `very-secret`:
@ -234,16 +233,18 @@ kafka-configs.sh --bootstrap-server localhost:9092 --alter --topic test --add-co
kafka-configs.sh --bootstrap-server localhost:9092 --alter --topic test --add-config cleanup.policy=[delete,compact]
```
Da als Key eines Records die (pseudonymisierte) Patienten-ID und die (anonymisierte) Erkrankungs-ID verwendet wird,
stehen mit obiger Konfiguration der Kafka-Topics nach 10 Sekunden nur noch der jeweils letzte Eintrag für den entsprechenden
Key zur Verfügung.
Da als Key eines Records die (pseudonymisierte) Patienten-ID verwendet wird, stehen mit obiger Konfiguration
der Kafka-Topics nach 10 Sekunden nur noch der jeweils letzte Eintrag für den entsprechenden Key zur Verfügung.
Da der Key sowohl für die Records in Richtung bwHC-Backend für die Rückantwort identisch aufgebaut ist, lassen sich so
auch im Falle eines Consent-Widerspruchs die enthaltenen Daten als auch die Offenlegung durch Verifikationsdaten in der
Antwort effektiv verhindern, da diese nach 10 Sekunden gelöscht werden.
Es steht dann nur noch die jeweils letzten Information zur Verfügung, dass für einen Patienten/eine Erkrankung
ein Consent-Widerspruch erfolgte.
Dieses Vorgehen empfiehlt sich, wenn Sie gespeicherte Records nachgelagert für andere Auswertungen verwenden möchten.
## Docker-Images
Diese Anwendung ist auch als Docker-Image verfügbar: https://github.com/CCC-MF/etl-processor/pkgs/container/etl-processor
@ -305,7 +306,7 @@ auf Docker-Compose mit der gestartet werden kann.
Die Anwendung verarbeitet `X-Forwarded`-HTTP-Header und kann daher auch hinter einem Reverse-Proxy betrieben werden.
Dabei werden, je nachdem welche Header durch den Reverse-Proxy gesendet werden auch Protokoll, Host oder auch Path-Prefix
Dabei werden, je nachdem welche Header durch den Reverse-Proxy gesendet werden auch Protokoll, Host oder auch Path-Präfix
automatisch erkannt und verwendet werden. Dadurch ist z.B. eine abweichende Angabe des Pfads problemlos möglich.
#### Beispiel *Traefik* (mit Docker-Labels):

View File

@ -4,23 +4,23 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
plugins {
war
id("org.springframework.boot") version "3.2.3"
id("io.spring.dependency-management") version "1.1.4"
kotlin("jvm") version "1.9.22"
kotlin("plugin.spring") version "1.9.22"
id("org.springframework.boot") version "3.2.12"
id("io.spring.dependency-management") version "1.1.5"
kotlin("jvm") version "1.9.24"
kotlin("plugin.spring") version "1.9.24"
}
group = "de.ukw.ccc"
version = "0.8.0"
group = "dev.dnpm"
version = "0.9.9"
var versions = mapOf(
"bwhc-dto-java" to "0.2.0",
"hapi-fhir" to "6.10.2",
"httpclient5" to "5.2.1",
"mockito-kotlin" to "5.2.1",
"bwhc-dto-java" to "0.3.0",
"hapi-fhir" to "6.10.5",
"httpclient5" to "5.2.3",
"mockito-kotlin" to "5.3.1",
// Webjars
"echarts" to "5.4.3",
"htmx.org" to "1.9.10"
"htmx.org" to "1.9.12"
)
java {
@ -70,7 +70,7 @@ dependencies {
implementation("ca.uhn.hapi.fhir:hapi-fhir-structures-r4:${versions["hapi-fhir"]}")
implementation("org.apache.httpcomponents.client5:httpclient5:${versions["httpclient5"]}")
implementation("com.jayway.jsonpath:json-path")
implementation("org.webjars:webjars-locator:0.50")
implementation("org.webjars:webjars-locator:0.52")
implementation("org.webjars.npm:echarts:${versions["echarts"]}")
implementation("org.webjars.npm:htmx.org:${versions["htmx.org"]}")
runtimeOnly("org.mariadb.jdbc:mariadb-java-client")
@ -85,6 +85,8 @@ dependencies {
testImplementation("org.mockito.kotlin:mockito-kotlin:${versions["mockito-kotlin"]}")
integrationTestImplementation("org.testcontainers:junit-jupiter")
integrationTestImplementation("org.testcontainers:postgresql")
// Override dependency version from org.testcontainers:junit-jupiter - CVE-2024-26308, CVE-2024-25710
integrationTestImplementation("org.apache.commons:commons-compress:1.26.2")
}
tasks.withType<KotlinCompile> {
@ -111,7 +113,7 @@ task<Test>("integrationTest") {
}
tasks.named<BootBuildImage>("bootBuildImage") {
imageName.set("ghcr.io/ccc-mf/etl-processor")
imageName.set("ghcr.io/pcvolkmer/etl-processor")
// Binding for CA Certs
bindings.set(listOf(
@ -121,7 +123,7 @@ tasks.named<BootBuildImage>("bootBuildImage") {
environment.set(environment.get() + mapOf(
// Enable this line to embed CA Certs into image on build time
//"BP_EMBED_CERTS" to "true",
"BP_OCI_SOURCE" to "https://github.com/CCC-MF/etl-processor",
"BP_OCI_SOURCE" to "https://github.com/pcvolkmer/etl-processor",
"BP_OCI_LICENSES" to "AGPLv3",
"BP_OCI_DESCRIPTION" to "ETL Processor for bwHC MTB files"
))

Binary file not shown.

Before

Width:  |  Height:  |  Size: 75 KiB

After

Width:  |  Height:  |  Size: 120 KiB

View File

@ -19,11 +19,14 @@
package dev.dnpm.etl.processor.web
import dev.dnpm.etl.processor.config.AppConfiguration
import dev.dnpm.etl.processor.config.AppSecurityConfiguration
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.RestConnectionCheckService
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.pseudonym.Generator
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.dnpm.etl.processor.services.TokenRepository
import dev.dnpm.etl.processor.services.TransformationService
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
@ -50,6 +53,7 @@ abstract class MockSink : Sinks.Many<Boolean>
@ContextConfiguration(
classes = [
ConfigController::class,
AppConfiguration::class,
AppSecurityConfiguration::class
]
)
@ -67,7 +71,9 @@ abstract class MockSink : Sinks.Many<Boolean>
MtbFileSender::class,
ConnectionCheckService::class,
RequestProcessor::class,
TransformationService::class
TransformationService::class,
TokenRepository::class,
RestConnectionCheckService::class
)
class ConfigControllerTest {

View File

@ -67,11 +67,13 @@ public class GpasPseudonymGenerator implements Generator {
private final RetryTemplate retryTemplate;
private final Logger log = LoggerFactory.getLogger(GpasPseudonymGenerator.class);
private final RestTemplate restTemplate;
private SSLContext customSslContext;
private RestTemplate restTemplate;
public GpasPseudonymGenerator(GPasConfigProperties gpasCfg, RetryTemplate retryTemplate) {
this.retryTemplate = retryTemplate;
this.restTemplate = getRestTemplete();
this.gPasUrl = gpasCfg.getUri();
this.psnTargetDomain = gpasCfg.getTarget();
@ -139,7 +141,6 @@ public class GpasPseudonymGenerator implements Generator {
HttpEntity<String> requestEntity = new HttpEntity<>(gPasRequestBody, this.httpHeader);
ResponseEntity<String> responseEntity;
var restTemplate = getRestTemplete();
try {
responseEntity = retryTemplate.execute(
@ -226,14 +227,8 @@ public class GpasPseudonymGenerator implements Generator {
}
protected RestTemplate getRestTemplete() {
if (restTemplate != null) {
return restTemplate;
}
if (customSslContext == null) {
restTemplate = new RestTemplate();
return restTemplate;
return new RestTemplate();
}
final var sslsf = new SSLConnectionSocketFactory(customSslContext);
final Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
@ -246,7 +241,6 @@ public class GpasPseudonymGenerator implements Generator {
final HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(
httpClient);
restTemplate = new RestTemplate(requestFactory);
return restTemplate;
return new RestTemplate(requestFactory);
}
}

View File

@ -20,7 +20,7 @@
package dev.dnpm.etl.processor.config
import com.fasterxml.jackson.databind.ObjectMapper
import dev.dnpm.etl.processor.monitoring.ReportService
import dev.dnpm.etl.processor.monitoring.*
import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator
import dev.dnpm.etl.processor.pseudonym.Generator
import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator
@ -44,6 +44,7 @@ import org.springframework.retry.support.RetryTemplateBuilder
import org.springframework.scheduling.annotation.EnableScheduling
import org.springframework.security.crypto.password.PasswordEncoder
import org.springframework.security.provisioning.InMemoryUserDetailsManager
import org.springframework.web.client.RestTemplate
import reactor.core.publisher.Sinks
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@ -62,6 +63,11 @@ class AppConfiguration {
private val logger = LoggerFactory.getLogger(AppConfiguration::class.java)
@Bean
fun restTemplate(): RestTemplate {
return RestTemplate()
}
@ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "GPAS")
@Bean
fun gpasPseudonymGenerator(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate): Generator {
@ -142,8 +148,29 @@ class AppConfiguration {
}
@Bean
fun configsUpdateProducer(): Sinks.Many<Boolean> {
return Sinks.many().multicast().directBestEffort()
fun connectionCheckUpdateProducer(): Sinks.Many<ConnectionCheckResult> {
return Sinks.many().multicast().onBackpressureBuffer()
}
@ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "GPAS")
@Bean
fun gPasConnectionCheckService(
restTemplate: RestTemplate,
gPasConfigProperties: GPasConfigProperties,
connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
): ConnectionCheckService {
return GPasConnectionCheckService(restTemplate, gPasConfigProperties, connectionCheckUpdateProducer)
}
@ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS")
@ConditionalOnMissingBean
@Bean
fun gPasConnectionCheckServiceOnDeprecatedProperty(
restTemplate: RestTemplate,
gPasConfigProperties: GPasConfigProperties,
connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
): ConnectionCheckService {
return GPasConnectionCheckService(restTemplate, gPasConfigProperties, connectionCheckUpdateProducer)
}
}

View File

@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.config
import com.fasterxml.jackson.databind.ObjectMapper
import dev.dnpm.etl.processor.input.KafkaInputListener
import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.KafkaConnectionCheckService
import dev.dnpm.etl.processor.output.KafkaMtbFileSender
@ -105,8 +106,11 @@ class AppKafkaConfiguration {
}
@Bean
fun connectionCheckService(consumerFactory: ConsumerFactory<String, String>, configsUpdateProducer: Sinks.Many<Boolean>): ConnectionCheckService {
return KafkaConnectionCheckService(consumerFactory.createConsumer(), configsUpdateProducer)
fun kafkaConnectionCheckService(
consumerFactory: ConsumerFactory<String, String>,
connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
): ConnectionCheckService {
return KafkaConnectionCheckService(consumerFactory.createConsumer(), connectionCheckUpdateProducer)
}
}

View File

@ -19,6 +19,7 @@
package dev.dnpm.etl.processor.config
import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.RestConnectionCheckService
import dev.dnpm.etl.processor.output.MtbFileSender
@ -47,11 +48,6 @@ class AppRestConfiguration {
private val logger = LoggerFactory.getLogger(AppRestConfiguration::class.java)
@Bean
fun restTemplate(): RestTemplate {
return RestTemplate()
}
@Bean
fun restMtbFileSender(
restTemplate: RestTemplate,
@ -63,12 +59,12 @@ class AppRestConfiguration {
}
@Bean
fun connectionCheckService(
fun restConnectionCheckService(
restTemplate: RestTemplate,
restTargetProperties: RestTargetProperties,
configsUpdateProducer: Sinks.Many<Boolean>
connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
): ConnectionCheckService {
return RestConnectionCheckService(restTemplate, restTargetProperties, configsUpdateProducer)
return RestConnectionCheckService(restTemplate, restTargetProperties, connectionCheckUpdateProducer)
}
}

View File

@ -35,12 +35,27 @@ class KafkaInputListener(
override fun onMessage(data: ConsumerRecord<String, String>) {
val mtbFile = objectMapper.readValue(data.value(), MtbFile::class.java)
val firstRequestIdHeader = data.headers().headers("requestId")?.firstOrNull()
val requestId = if (null != firstRequestIdHeader) {
String(firstRequestIdHeader.value())
} else {
""
}
if (mtbFile.consent.status == Consent.Status.ACTIVE) {
logger.debug("Accepted MTB File for processing")
requestProcessor.processMtbFile(mtbFile)
if (requestId.isBlank()) {
requestProcessor.processMtbFile(mtbFile)
} else {
requestProcessor.processMtbFile(mtbFile, requestId)
}
} else {
logger.debug("Accepted MTB File and process deletion")
requestProcessor.processDeletion(mtbFile.patient.id)
if (requestId.isBlank()) {
requestProcessor.processDeletion(mtbFile.patient.id)
} else {
requestProcessor.processDeletion(mtbFile.patient.id, requestId)
}
}
}
}

View File

@ -20,14 +20,21 @@
package dev.dnpm.etl.processor.monitoring
import dev.dnpm.etl.processor.config.GPasConfigProperties
import dev.dnpm.etl.processor.config.RestTargetProperties
import jakarta.annotation.PostConstruct
import org.apache.kafka.clients.consumer.Consumer
import org.apache.kafka.common.errors.TimeoutException
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.http.HttpEntity
import org.springframework.http.HttpHeaders
import org.springframework.http.HttpMethod
import org.springframework.http.HttpStatus
import org.springframework.http.MediaType
import org.springframework.http.RequestEntity
import org.springframework.scheduling.annotation.Scheduled
import org.springframework.web.client.RestTemplate
import org.springframework.web.util.UriComponentsBuilder
import reactor.core.publisher.Sinks
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@ -38,11 +45,22 @@ interface ConnectionCheckService {
}
interface OutputConnectionCheckService : ConnectionCheckService
sealed class ConnectionCheckResult {
abstract val available: Boolean
data class KafkaConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult()
data class RestConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult()
data class GPasConnectionCheckResult(override val available: Boolean) : ConnectionCheckResult()
}
class KafkaConnectionCheckService(
private val consumer: Consumer<String, String>,
@Qualifier("configsUpdateProducer")
private val configsUpdateProducer: Sinks.Many<Boolean>
) : ConnectionCheckService {
@Qualifier("connectionCheckUpdateProducer")
private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
) : OutputConnectionCheckService {
private var connectionAvailable: Boolean = false
@ -55,7 +73,10 @@ class KafkaConnectionCheckService(
} catch (e: TimeoutException) {
false
}
configsUpdateProducer.emitNext(connectionAvailable, Sinks.EmitFailureHandler.FAIL_FAST)
connectionCheckUpdateProducer.emitNext(
ConnectionCheckResult.KafkaConnectionCheckResult(connectionAvailable),
Sinks.EmitFailureHandler.FAIL_FAST
)
}
override fun connectionAvailable(): Boolean {
@ -67,9 +88,9 @@ class KafkaConnectionCheckService(
class RestConnectionCheckService(
private val restTemplate: RestTemplate,
private val restTargetProperties: RestTargetProperties,
@Qualifier("configsUpdateProducer")
private val configsUpdateProducer: Sinks.Many<Boolean>
) : ConnectionCheckService {
@Qualifier("connectionCheckUpdateProducer")
private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
) : OutputConnectionCheckService {
private var connectionAvailable: Boolean = false
@ -84,7 +105,55 @@ class RestConnectionCheckService(
} catch (e: Exception) {
false
}
configsUpdateProducer.emitNext(connectionAvailable, Sinks.EmitFailureHandler.FAIL_FAST)
connectionCheckUpdateProducer.emitNext(
ConnectionCheckResult.RestConnectionCheckResult(connectionAvailable),
Sinks.EmitFailureHandler.FAIL_FAST
)
}
override fun connectionAvailable(): Boolean {
return this.connectionAvailable
}
}
class GPasConnectionCheckService(
private val restTemplate: RestTemplate,
private val gPasConfigProperties: GPasConfigProperties,
@Qualifier("connectionCheckUpdateProducer")
private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>
) : ConnectionCheckService {
private var connectionAvailable: Boolean = false
@PostConstruct
@Scheduled(cron = "0 * * * * *")
fun check() {
connectionAvailable = try {
val uri = UriComponentsBuilder.fromUriString(
gPasConfigProperties.uri?.replace("/\$pseudonymizeAllowCreate", "/\$pseudonymize").toString()
)
.queryParam("target", gPasConfigProperties.target)
.queryParam("original", "???")
.build().toUri()
val headers = HttpHeaders()
headers.contentType = MediaType.APPLICATION_JSON
if (!gPasConfigProperties.username.isNullOrBlank() && !gPasConfigProperties.password.isNullOrBlank()) {
headers.setBasicAuth(gPasConfigProperties.username, gPasConfigProperties.password)
}
restTemplate.exchange(
uri,
HttpMethod.GET,
HttpEntity<Void>(headers),
Void::class.java
).statusCode == HttpStatus.OK
} catch (e: Exception) {
false
}
connectionCheckUpdateProducer.emitNext(
ConnectionCheckResult.GPasConnectionCheckResult(connectionAvailable),
Sinks.EmitFailureHandler.FAIL_FAST
)
}
override fun connectionAvailable(): Boolean {

View File

@ -57,6 +57,7 @@ class ReportService(
data class Issue(val severity: Severity, val message: String)
enum class Severity(@JsonValue val value: String) {
FATAL("fatal"),
ERROR("error"),
WARNING("warning"),
INFO("info")

View File

@ -94,8 +94,7 @@ class KafkaMtbFileSender(
}
private fun key(request: MtbFileSender.MtbFileRequest): String {
return "{\"pid\": \"${request.mtbFile.patient.id}\", " +
"\"eid\": \"${request.mtbFile.episode.id}\"}"
return "{\"pid\": \"${request.mtbFile.patient.id}\"}"
}
private fun key(request: MtbFileSender.DeleteRequest): String {

View File

@ -33,4 +33,8 @@ class PseudonymizeService(
}
}
fun prefix(): String {
return configProperties.prefix
}
}

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -20,34 +20,206 @@
package dev.dnpm.etl.processor.pseudonym
import de.ukw.ccc.bwhc.dto.MtbFile
import org.apache.commons.codec.digest.DigestUtils
/** Replaces patient ID with generated patient pseudonym
*
* @param pseudonymizeService The pseudonymizeService to be used
*
* @return The MTB file containing patient pseudonymes
*/
infix fun MtbFile.pseudonymizeWith(pseudonymizeService: PseudonymizeService) {
val patientPseudonym = pseudonymizeService.patientPseudonym(this.patient.id)
this.episode.patient = patientPseudonym
this.carePlans.forEach { it.patient = patientPseudonym }
this.episode?.patient = patientPseudonym
this.carePlans?.forEach { it.patient = patientPseudonym }
this.patient.id = patientPseudonym
this.claims.forEach { it.patient = patientPseudonym }
this.consent.patient = patientPseudonym
this.claimResponses.forEach { it.patient = patientPseudonym }
this.diagnoses.forEach { it.patient = patientPseudonym }
this.ecogStatus.forEach { it.patient = patientPseudonym }
this.familyMemberDiagnoses.forEach { it.patient = patientPseudonym }
this.geneticCounsellingRequests.forEach { it.patient = patientPseudonym }
this.histologyReevaluationRequests.forEach { it.patient = patientPseudonym }
this.histologyReports.forEach {
this.claims?.forEach { it.patient = patientPseudonym }
this.consent?.patient = patientPseudonym
this.claimResponses?.forEach { it.patient = patientPseudonym }
this.diagnoses?.forEach { it.patient = patientPseudonym }
this.ecogStatus?.forEach { it.patient = patientPseudonym }
this.familyMemberDiagnoses?.forEach { it.patient = patientPseudonym }
this.geneticCounsellingRequests?.forEach { it.patient = patientPseudonym }
this.histologyReevaluationRequests?.forEach { it.patient = patientPseudonym }
this.histologyReports?.forEach {
it.patient = patientPseudonym
it.tumorMorphology.patient = patientPseudonym
it.tumorMorphology?.patient = patientPseudonym
}
this.lastGuidelineTherapies?.forEach { it.patient = patientPseudonym }
this.molecularPathologyFindings?.forEach { it.patient = patientPseudonym }
this.molecularTherapies?.forEach { molecularTherapy -> molecularTherapy.history.forEach { it.patient = patientPseudonym } }
this.ngsReports?.forEach { it.patient = patientPseudonym }
this.previousGuidelineTherapies?.forEach { it.patient = patientPseudonym }
this.rebiopsyRequests?.forEach { it.patient = patientPseudonym }
this.recommendations?.forEach { it.patient = patientPseudonym }
this.responses?.forEach { it.patient = patientPseudonym }
this.studyInclusionRequests?.forEach { it.patient = patientPseudonym }
this.specimens?.forEach { it.patient = patientPseudonym }
}
/**
* Creates new hash of content IDs with given prefix except for patient IDs
*
* @param pseudonymizeService The pseudonymizeService to be used
*
* @return The MTB file containing rehashed content IDs
*/
infix fun MtbFile.anonymizeContentWith(pseudonymizeService: PseudonymizeService) {
val prefix = pseudonymizeService.prefix()
fun anonymize(id: String): String {
val hash = DigestUtils.sha256Hex("$prefix-$id").substring(0, 41).lowercase()
return "$prefix$hash"
}
this.episode?.apply {
id = id?.let {
anonymize(it)
}
}
this.carePlans?.onEach { carePlan ->
carePlan?.apply {
id = id?.let { anonymize(it) }
diagnosis = diagnosis?.let { anonymize(it) }
geneticCounsellingRequest = geneticCounsellingRequest?.let { anonymize(it) }
rebiopsyRequests = rebiopsyRequests.map { it?.let { anonymize(it) } }
recommendations = recommendations.map { it?.let { anonymize(it) } }
studyInclusionRequests = studyInclusionRequests.map { it?.let { anonymize(it) } }
}
}
this.claims?.onEach { claim ->
claim?.apply {
id = id?.let { anonymize(it) }
therapy = therapy?.let { anonymize(it) }
}
}
this.claimResponses?.onEach { claimResponse ->
claimResponse?.apply {
id = id?.let { anonymize(it) }
claim = claim?.let { anonymize(it) }
}
}
this.consent?.apply {
id = id?.let { anonymize(it) }
}
this.diagnoses?.onEach { diagnosis ->
diagnosis?.apply {
id = id?.let { anonymize(it) }
histologyResults = histologyResults?.map { it?.let { anonymize(it) } }
}
}
this.ecogStatus?.onEach { ecogStatus ->
ecogStatus?.apply {
id = id?.let { anonymize(it) }
}
}
this.familyMemberDiagnoses?.onEach { familyMemberDiagnosis ->
familyMemberDiagnosis?.apply {
id = id?.let { anonymize(it) }
}
}
this.geneticCounsellingRequests?.onEach { geneticCounsellingRequest ->
geneticCounsellingRequest?.apply {
id = id?.let { anonymize(it) }
}
}
this.histologyReevaluationRequests?.onEach { histologyReevaluationRequest ->
histologyReevaluationRequest?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
}
this.histologyReports?.onEach { histologyReport ->
histologyReport?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
tumorMorphology?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
tumorCellContent?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
}
}
this.lastGuidelineTherapies?.onEach { lastGuidelineTherapy ->
lastGuidelineTherapy?.apply {
id = id?.let { anonymize(it) }
diagnosis = diagnosis?.let { anonymize(it) }
}
}
this.molecularPathologyFindings?.onEach { molecularPathologyFinding ->
molecularPathologyFinding?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
}
this.molecularTherapies?.onEach { molecularTherapy ->
molecularTherapy?.apply {
history?.onEach { history ->
history?.apply {
id = id?.let { anonymize(it) }
basedOn = basedOn?.let { anonymize(it) }
}
}
}
}
this.ngsReports?.onEach { ngsReport ->
ngsReport?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
tumorCellContent?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
simpleVariants?.onEach { simpleVariant ->
simpleVariant?.apply {
id = id?.let { anonymize(it) }
}
}
}
}
this.previousGuidelineTherapies?.onEach { previousGuidelineTherapy ->
previousGuidelineTherapy?.apply {
id = id?.let { anonymize(it) }
diagnosis = diagnosis?.let { anonymize(it) }
medication.forEach { medication ->
medication?.apply {
id = id?.let { anonymize(it) }
}
}
}
}
this.rebiopsyRequests?.onEach { rebiopsyRequest ->
rebiopsyRequest?.apply {
id = id?.let { anonymize(it) }
specimen = specimen?.let { anonymize(it) }
}
}
this.recommendations?.onEach { recommendation ->
recommendation?.apply {
id = id?.let { anonymize(it) }
diagnosis = diagnosis?.let { anonymize(it) }
ngsReport = ngsReport?.let { anonymize(it) }
}
}
this.responses?.onEach { response ->
response?.apply {
id = id?.let { anonymize(it) }
therapy = therapy?.let { anonymize(it) }
}
}
this.studyInclusionRequests?.onEach { studyInclusionRequest ->
studyInclusionRequest?.apply {
id = id?.let { anonymize(it) }
reason = reason?.let { anonymize(it) }
}
}
this.specimens?.onEach { specimen ->
specimen?.apply {
id = id?.let { anonymize(it) }
}
}
this.lastGuidelineTherapies.forEach { it.patient = patientPseudonym }
this.molecularPathologyFindings.forEach { it.patient = patientPseudonym }
this.molecularTherapies.forEach { molecularTherapy -> molecularTherapy.history.forEach { it.patient = patientPseudonym } }
this.ngsReports.forEach { it.patient = patientPseudonym }
this.previousGuidelineTherapies.forEach { it.patient = patientPseudonym }
this.rebiopsyRequests.forEach { it.patient = patientPseudonym }
this.recommendations.forEach { it.patient = patientPseudonym }
this.recommendations.forEach { it.patient = patientPseudonym }
this.responses.forEach { it.patient = patientPseudonym }
this.studyInclusionRequests.forEach { it.patient = patientPseudonym }
this.specimens.forEach { it.patient = patientPseudonym }
}

View File

@ -28,6 +28,7 @@ import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
import dev.dnpm.etl.processor.pseudonym.anonymizeContentWith
import dev.dnpm.etl.processor.pseudonym.pseudonymizeWith
import org.apache.commons.codec.binary.Base32
import org.apache.commons.codec.digest.DigestUtils
@ -48,10 +49,14 @@ class RequestProcessor(
) {
fun processMtbFile(mtbFile: MtbFile) {
val requestId = UUID.randomUUID().toString()
processMtbFile(mtbFile, UUID.randomUUID().toString())
}
fun processMtbFile(mtbFile: MtbFile, requestId: String) {
val pid = mtbFile.patient.id
mtbFile pseudonymizeWith pseudonymizeService
mtbFile anonymizeContentWith pseudonymizeService
val request = MtbFileSender.MtbFileRequest(requestId, transformationService.transform(mtbFile))
@ -103,8 +108,10 @@ class RequestProcessor(
}
fun processDeletion(patientId: String) {
val requestId = UUID.randomUUID().toString()
processDeletion(patientId, UUID.randomUUID().toString())
}
fun processDeletion(patientId: String, requestId: String) {
try {
val patientPseudonym = pseudonymizeService.patientPseudonym(patientId)

View File

@ -19,7 +19,10 @@
package dev.dnpm.etl.processor.web
import dev.dnpm.etl.processor.monitoring.ConnectionCheckResult
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.GPasConnectionCheckService
import dev.dnpm.etl.processor.monitoring.OutputConnectionCheckService
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.pseudonym.Generator
import dev.dnpm.etl.processor.security.Role
@ -40,22 +43,29 @@ import reactor.core.publisher.Sinks
@Controller
@RequestMapping(path = ["configs"])
class ConfigController(
@Qualifier("configsUpdateProducer")
private val configsUpdateProducer: Sinks.Many<Boolean>,
@Qualifier("connectionCheckUpdateProducer")
private val connectionCheckUpdateProducer: Sinks.Many<ConnectionCheckResult>,
private val transformationService: TransformationService,
private val pseudonymGenerator: Generator,
private val mtbFileSender: MtbFileSender,
private val connectionCheckService: ConnectionCheckService,
private val connectionCheckServices: List<ConnectionCheckService>,
private val tokenService: TokenService?,
private val userRoleService: UserRoleService?
) {
@GetMapping
fun index(model: Model): String {
val outputConnectionAvailable =
connectionCheckServices.filterIsInstance<OutputConnectionCheckService>().first().connectionAvailable()
val gPasConnectionAvailable =
connectionCheckServices.filterIsInstance<GPasConnectionCheckService>().firstOrNull()?.connectionAvailable()
model.addAttribute("pseudonymGenerator", pseudonymGenerator.javaClass.simpleName)
model.addAttribute("mtbFileSender", mtbFileSender.javaClass.simpleName)
model.addAttribute("mtbFileEndpoint", mtbFileSender.endpoint())
model.addAttribute("connectionAvailable", connectionCheckService.connectionAvailable())
model.addAttribute("outputConnectionAvailable", outputConnectionAvailable)
model.addAttribute("gPasConnectionAvailable", gPasConnectionAvailable)
model.addAttribute("tokensEnabled", tokenService != null)
if (tokenService != null) {
model.addAttribute("tokens", tokenService.findAll())
@ -73,11 +83,14 @@ class ConfigController(
return "configs"
}
@GetMapping(params = ["connectionAvailable"])
fun connectionAvailable(model: Model): String {
@GetMapping(params = ["outputConnectionAvailable"])
fun outputConnectionAvailable(model: Model): String {
val outputConnectionAvailable =
connectionCheckServices.filterIsInstance<OutputConnectionCheckService>().first().connectionAvailable()
model.addAttribute("mtbFileSender", mtbFileSender.javaClass.simpleName)
model.addAttribute("mtbFileEndpoint", mtbFileSender.endpoint())
model.addAttribute("connectionAvailable", connectionCheckService.connectionAvailable())
model.addAttribute("outputConnectionAvailable", outputConnectionAvailable)
if (tokenService != null) {
model.addAttribute("tokensEnabled", true)
model.addAttribute("tokens", tokenService.findAll())
@ -85,7 +98,25 @@ class ConfigController(
model.addAttribute("tokens", listOf<Token>())
}
return "configs/connectionAvailable"
return "configs/outputConnectionAvailable"
}
@GetMapping(params = ["gPasConnectionAvailable"])
fun gPasConnectionAvailable(model: Model): String {
val gPasConnectionAvailable =
connectionCheckServices.filterIsInstance<GPasConnectionCheckService>().firstOrNull()?.connectionAvailable()
model.addAttribute("mtbFileSender", mtbFileSender.javaClass.simpleName)
model.addAttribute("mtbFileEndpoint", mtbFileSender.endpoint())
model.addAttribute("gPasConnectionAvailable", gPasConnectionAvailable)
if (tokenService != null) {
model.addAttribute("tokensEnabled", true)
model.addAttribute("tokens", tokenService.findAll())
} else {
model.addAttribute("tokens", listOf<Token>())
}
return "configs/gPasConnectionAvailable"
}
@PostMapping(path = ["tokens"])
@ -152,9 +183,15 @@ class ConfigController(
@GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE])
fun events(): Flux<ServerSentEvent<Any>> {
return configsUpdateProducer.asFlux().map {
return connectionCheckUpdateProducer.asFlux().map {
val event = when (it) {
is ConnectionCheckResult.KafkaConnectionCheckResult -> "output-connection-check"
is ConnectionCheckResult.RestConnectionCheckResult -> "output-connection-check"
is ConnectionCheckResult.GPasConnectionCheckResult -> "gpas-connection-check"
}
ServerSentEvent.builder<Any>()
.event("connection-available").id("none").data("")
.event(event).id("none").data(it)
.build()
}
}

View File

@ -134,7 +134,6 @@ class StatisticsRestController(
@GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE])
fun updater(): Flux<ServerSentEvent<Any>> {
return statisticsUpdateProducer.asFlux().flatMap {
println(it)
Flux.fromIterable(
listOf(
ServerSentEvent.builder<Any>()

View File

@ -257,6 +257,10 @@ form.samplecode-input input:focus-visible {
display: block;
}
.userrole-form {
display: inline-block;
}
.userrole-form form {
margin: 0;
padding: 0;
@ -321,6 +325,15 @@ table {
font-family: sans-serif;
}
table.config-table td:first-child {
width: 24em;
min-width: fit-content;
}
table.config-table td > button:last-of-type {
float: right;
}
.border > table {
padding: 0;
border: none;
@ -490,7 +503,7 @@ td.clipboard.clipped {
.btn:active,
.btn:hover {
filter: drop-shadow(1px 1px 1px gray) var(--dark);
filter: drop-shadow(0px 1px 1px var(--bg-gray)) var(--dark);
}
.btn:active {
@ -555,15 +568,24 @@ input.inline:focus-visible {
font-weight: bold;
}
.chart {
width: calc(100% - 2.4rem - 4px);
height: 320px;
display: inline-block;
.charts {
display: grid;
grid-gap: 1em;
grid-template:
"a b" 28em
"c c" 28em / 1fr 1fr;
}
.chart-50pc {
width: calc(50% - 2.4rem - 4px);
.charts > .grid-left {
grid-area: a;
}
.charts > .grid-right {
grid-area: b;
}
.charts > .grid-full {
grid-area: c;
}
.connection-display {
@ -571,7 +593,7 @@ input.inline:focus-visible {
grid-template-columns: 10em 16em 10em;
place-items: center;
width: fit-content;
margin: 1em 0;
margin: 1em auto;
}
.connection-display > * {
@ -609,6 +631,32 @@ input.inline:focus-visible {
color: var(--bg-red);
}
.tab {
padding: 1em;
border: none;
border-radius: 3px 3px 0 0;
cursor: pointer;
transition: all 0.2s;
font-weight: bold;
}
.tab:hover,
.tab.active {
background: var(--table-border);
}
.tabcontent {
border: 1px solid var(--table-border);
border-radius: 0 .5em .5em .5em;
display: none;
padding: 1em;
}
.tabcontent.active {
display: block;
}
a.reload {
display: none;
position: absolute;

View File

@ -10,90 +10,116 @@
<main>
<h1>Konfiguration</h1>
<section>
<h2>🔧 Allgemeine Konfiguration</h2>
<table>
<thead>
<tr>
<th>Name</th>
<th>Wert</th>
</tr>
</thead>
<tbody>
<div class="tabs">
<button class="tab active" onclick="selectTab(this, 'common');">Allgemeine Informationen</button>
<button class="tab" onclick="selectTab(this, 'security');">Sicherheit</button>
<button class="tab" onclick="selectTab(this, 'transformation');">Transformationen</button>
</div>
<div id="common" class="tabcontent active">
<section>
<h2>🔧 Allgemeine Konfiguration</h2>
<table class="config-table">
<thead>
<tr>
<td>Pseudonym erzeugt über</td>
<td>[[ ${pseudonymGenerator} ]]</td>
<th>Name</th>
<th>Wert</th>
</tr>
</thead>
<tbody>
<tr>
<td>Pseudonym erzeugt über</td>
<td>[[ ${pseudonymGenerator} ]]</td>
</tr>
<tr>
<td>MTBFile-Sender</td>
<td>[[ ${mtbFileSender} ]]</td>
</tr>
<tr>
<td th:if="${mtbFileSender.startsWith('Rest')}">REST-Endpunkt</td>
<td th:if="${mtbFileSender.startsWith('Kafka')}">Kafka-Broker und Topics</td>
<td>[[ ${mtbFileEndpoint} ]]</td>
</tr>
</tbody>
</table>
</section>
<section hx-ext="sse" th:sse-connect="@{/configs/events}">
<div th:insert="~{configs/gPasConnectionAvailable.html}" th:hx-get="@{/configs?gPasConnectionAvailable}" hx-trigger="sse:gpas-connection-check">
</div>
</section>
<section hx-ext="sse" th:sse-connect="@{/configs/events}">
<div th:insert="~{configs/outputConnectionAvailable.html}" th:hx-get="@{/configs?outputConnectionAvailable}" hx-trigger="sse:output-connection-check">
</div>
</section>
</div>
<div id="security" class="tabcontent">
<section th:insert="~{configs/tokens.html}">
</section>
<section th:insert="~{configs/userroles.html}">
</section>
</div>
<div id="transformation" class="tabcontent">
<section>
<h2><span th:if="${not transformations.isEmpty()}"></span><span th:if="${transformations.isEmpty()}"></span> Transformationen</h2>
<h3>Syntax</h3>
Hier einige Beispiele zum Syntax des JSON-Path
<ul>
<li style="padding: 0.6rem 0;"><span class="bg-path">diagnoses[*].icdO3T.version</span>: Ersetze die ICD-O3T-Version in allen Diagnosen, z.B. zur Version der deutschen Übersetzung</li>
<li style="padding: 0.6rem 0;"><span class="bg-path">patient.gender</span>: Ersetze das Geschlecht des Patienten, z.B. in das von bwHC verlangte Format</li>
</ul>
<h3>Konfigurierte Transformationen</h3>
<th:block th:if="${transformations.isEmpty()}">
<p>
Keine konfigurierten Transformationen.
</p>
</th:block>
<th:block th:if="${not transformations.isEmpty()}">
<p>
Hier sehen Sie eine Übersicht der konfigurierten Transformationen.
</p>
<table class="config-table">
<thead>
<tr>
<td>MTBFile-Sender</td>
<td>[[ ${mtbFileSender} ]]</td>
<th>JSON-Path</th>
<th>Transformation von &rArr; nach</th>
</tr>
<tr>
<td th:if="${mtbFileSender.startsWith('Rest')}">REST-Endpunkt</td>
<td th:if="${mtbFileSender.startsWith('Kafka')}">Kafka-Broker und Topics</td>
<td>[[ ${mtbFileEndpoint} ]]</td>
</thead>
<tbody>
<tr th:each="transformation : ${transformations}">
<td>
<span class="bg-path" title="Ersetze Wert(e) an dieser Stelle im MTB-File">[[ ${transformation.path} ]]</span>
</td>
<td>
<span class="bg-from" title="Ersetze immer dann, wenn dieser Wert enthalten ist">[[ ${transformation.existingValue} ]]</span>
<strong>&rArr;</strong>
<span class="bg-to" title="Ersetze durch diesen Wert">[[ ${transformation.newValue} ]]</span>
</td>
</tr>
</tbody>
</table>
</section>
<section th:insert="~{configs/tokens.html}">
</section>
<section th:insert="~{configs/userroles.html}">
</section>
<section hx-ext="sse" th:sse-connect="@{/configs/events}">
<div th:insert="~{configs/connectionAvailable.html}" th:hx-get="@{/configs?connectionAvailable}" hx-trigger="sse:connection-available">
</div>
</section>
<section>
<h2><span th:if="${not transformations.isEmpty()}"></span><span th:if="${transformations.isEmpty()}"></span> Transformationen</h2>
<h3>Syntax</h3>
Hier einige Beispiele zum Syntax des JSON-Path
<ul>
<li style="padding: 0.6rem 0;"><span class="bg-path">diagnoses[*].icdO3T.version</span>: Ersetze die ICD-O3T-Version in allen Diagnosen, z.B. zur Version der deutschen Übersetzung</li>
<li style="padding: 0.6rem 0;"><span class="bg-path">patient.gender</span>: Ersetze das Geschlecht des Patienten, z.B. in das von bwHC verlangte Format</li>
</ul>
<h3>Konfigurierte Transformationen</h3>
<th:block th:if="${transformations.isEmpty()}">
<p>
Keine konfigurierten Transformationen.
</p>
</th:block>
<th:block th:if="${not transformations.isEmpty()}">
<p>
Hier sehen Sie eine Übersicht der konfigurierten Transformationen.
</p>
<table>
<thead>
<tr>
<th>JSON-Path</th>
<th>Transformation von &rArr; nach</th>
</tr>
</thead>
<tbody>
<tr th:each="transformation : ${transformations}">
<td>
<span class="bg-path" title="Ersetze Wert(e) an dieser Stelle im MTB-File">[[ ${transformation.path} ]]</span>
</td>
<td>
<span class="bg-from" title="Ersetze immer dann, wenn dieser Wert enthalten ist">[[ ${transformation.existingValue} ]]</span>
<strong>&rArr;</strong>
<span class="bg-to" title="Ersetze durch diesen Wert">[[ ${transformation.newValue} ]]</span>
</td>
</tr>
</tbody>
</table>
</th:block>
</section>
</tbody>
</table>
</th:block>
</section>
</div>
</main>
<script th:src="@{/scripts.js}"></script>
<script th:src="@{/webjars/htmx.org/dist/htmx.min.js}"></script>
<script th:src="@{/webjars/htmx.org/dist/ext/sse.js}"></script>
<script>
function selectTab(self, elem) {
Array.from(document.getElementsByClassName('tab')).forEach(e => e.className = 'tab');
self.className = 'tab active';
Array.from(document.getElementsByClassName('tabcontent')).forEach(e => e.className = 'tabcontent');
document.getElementById(elem).className = 'tabcontent active';
}
</script>
</body>
</html>

View File

@ -0,0 +1,19 @@
<th:block th:if="${gPasConnectionAvailable == null}">
<h2><span>🟦</span> gPAS nicht konfiguriert - Patienten-IDs werden intern anonymisiert</h2>
</th:block>
<th:block th:if="${gPasConnectionAvailable != null}">
<h2><span th:if="${gPasConnectionAvailable}"></span><span th:if="${not(gPasConnectionAvailable)}"></span> Verbindung zu gPAS</h2>
<div>
Die Verbindung ist aktuell
<strong th:if="${gPasConnectionAvailable}" style="color: green">verfügbar.</strong>
<strong th:if="${not(gPasConnectionAvailable)}" style="color: red">nicht verfügbar.</strong>
</div>
<div class="connection-display border">
<img th:src="@{/server.png}" alt="ETL-Processor" />
<span class="connection" th:classappend="${gPasConnectionAvailable ? 'available' : ''}"></span>
<img th:src="@{/server.png}" alt="gPAS" />
<span>ETL-Processor</span>
<span></span>
<span>gPAS</span>
</div>
</th:block>

View File

@ -1,12 +1,12 @@
<h2><span th:if="${connectionAvailable}"></span><span th:if="${not(connectionAvailable)}"></span> Verbindung zum bwHC-Backend</h2>
<h2><span th:if="${outputConnectionAvailable}"></span><span th:if="${not(outputConnectionAvailable)}"></span> MTB-File Verbindung</h2>
<div>
Verbindung über <code>[[ ${mtbFileSender} ]]</code>. Die Verbindung ist aktuell
<strong th:if="${connectionAvailable}" style="color: green">verfügbar.</strong>
<strong th:if="${not(connectionAvailable)}" style="color: red">nicht verfügbar.</strong>
<strong th:if="${outputConnectionAvailable}" style="color: green">verfügbar.</strong>
<strong th:if="${not(outputConnectionAvailable)}" style="color: red">nicht verfügbar.</strong>
</div>
<div class="connection-display border">
<img th:src="@{/server.png}" alt="ETL-Processor" />
<span class="connection" th:classappend="${connectionAvailable ? 'available' : ''}"></span>
<span class="connection" th:classappend="${outputConnectionAvailable ? 'available' : ''}"></span>
<img th:if="${mtbFileSender.startsWith('Rest')}" th:src="@{/server.png}" alt="bwHC-Backend" />
<img th:if="${mtbFileSender.startsWith('Kafka')}" th:src="@{/kafka.png}" alt="Kafka-Broker" />
<span>ETL-Processor</span>

View File

@ -7,19 +7,20 @@
<h2><span></span> Tokens</h2>
<div class="border">
<div th:if="${tokens.isEmpty()}">Noch keine Tokens vorhanden.</div>
<table th:if="${not tokens.isEmpty()}">
<table th:if="${not tokens.isEmpty()}" class="config-table">
<thead>
<tr>
<th>Name</th>
<th>Erstellt</th>
<th></th>
</tr>
</thead>
<tbody>
<tr th:each="token : ${tokens}">
<td>[[ ${token.name} ]]</td>
<td><time th:datetime="${token.createdAt}">[[ ${token.createdAt} ]]</time></td>
<td><button class="btn btn-red" th:hx-delete="@{/configs/tokens/{id}(id=${token.id})}" hx-target="#tokens">Löschen</button></td>
<td>
<time th:datetime="${token.createdAt}">[[ ${token.createdAt} ]]</time>
<button class="btn btn-red" th:hx-delete="@{/configs/tokens/{id}(id=${token.id})}" hx-target="#tokens">Löschen</button>
</td>
</tr>
</tbody>
</table>

View File

@ -7,12 +7,11 @@
<h2><span></span> Benutzerberechtigungen</h2>
<div class="border">
<div th:if="${userRoles.isEmpty()}">Noch keine Benutzerberechtigungen vorhanden.</div>
<table th:if="${not userRoles.isEmpty()}">
<table th:if="${not userRoles.isEmpty()}" class="config-table">
<thead>
<tr>
<th>Benutzername</th>
<th>Rolle</th>
<th></th>
</tr>
</thead>
<tbody>
@ -29,8 +28,6 @@
<button class="btn btn-blue" th:disabled="${#authorization.authentication.getName() == userRole.username}">Übernehmen</button>
</form>
</div>
</td>
<td>
<button class="btn btn-red" th:hx-delete="@{/configs/userroles/{id}(id=${userRole.id})}" hx-target="#userroles" th:disabled="${#authorization.authentication.getName() == userRole.username}">Löschen</button>
</td>
</tr>

View File

@ -55,6 +55,7 @@
<td th:if="${issue.severity.value == 'info'}" class="bg-blue"><small>[[ ${issue.severity} ]]</small></td>
<td th:if="${issue.severity.value == 'warning'}" class="bg-yellow"><small>[[ ${issue.severity} ]]</small></td>
<td th:if="${issue.severity.value == 'error'}" class="bg-red"><small>[[ ${issue.severity} ]]</small></td>
<td th:if="${issue.severity.value == 'fatal'}" class="bg-red"><small>[[ ${issue.severity} ]]</small></td>
<td>[[ ${issue.message} ]]</td>
</tr>
</tbody>

View File

@ -18,11 +18,11 @@
<p>
Anfragen zur Aktualisierung von Patientendaten durch Übermittlung eines MTB-Files.
</p>
<div>
<div id="piechart1" class="chart chart-50pc"></div>
<div id="piechart2" class="chart chart-50pc"></div>
<div class="charts">
<div id="piechart1" class="chart grid-left"></div>
<div id="piechart2" class="chart grid-right"></div>
<div id="barchart" class="chart grid-full"></div>
</div>
<div id="barchart" class="chart"></div>
</section>
<section>
@ -30,11 +30,11 @@
<p>
Anfragen zur Löschung von Patientendaten, wenn kein Consent vorliegt.
</p>
<div>
<div id="piechartdel1" class="chart chart-50pc"></div>
<div id="piechartdel2" class="chart chart-50pc"></div>
<div class="charts">
<div id="piechartdel1" class="chart grid-left"></div>
<div id="piechartdel2" class="chart grid-right"></div>
<div id="barchartdel" class="chart grid-full"></div>
</div>
<div id="barchartdel" class="chart"></div>
</section>
</main>

View File

@ -25,6 +25,9 @@ import de.ukw.ccc.bwhc.dto.MtbFile
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.services.RequestProcessor
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.common.header.internals.RecordHeader
import org.apache.kafka.common.header.internals.RecordHeaders
import org.apache.kafka.common.record.TimestampType
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
@ -34,6 +37,7 @@ import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.any
import org.mockito.kotlin.times
import org.mockito.kotlin.verify
import java.util.*
@ExtendWith(MockitoExtension::class)
class KafkaInputListenerTest {
@ -76,4 +80,33 @@ class KafkaInputListenerTest {
verify(requestProcessor, times(1)).processDeletion(anyString())
}
@Test
fun shouldProcessMtbFileRequestWithExistingRequestId() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.ACTIVE).build())
.build()
val headers = RecordHeaders(listOf(RecordHeader("requestId", UUID.randomUUID().toString().toByteArray())))
kafkaInputListener.onMessage(
ConsumerRecord("testtopic", 0, 0, -1L, TimestampType.NO_TIMESTAMP_TYPE, -1, -1, "", this.objectMapper.writeValueAsString(mtbFile), headers, Optional.empty())
)
verify(requestProcessor, times(1)).processMtbFile(any(), anyString())
}
@Test
fun shouldProcessDeleteRequestWithExistingRequestId() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
.build()
val headers = RecordHeaders(listOf(RecordHeader("requestId", UUID.randomUUID().toString().toByteArray())))
kafkaInputListener.onMessage(
ConsumerRecord("testtopic", 0, 0, -1L, TimestampType.NO_TIMESTAMP_TYPE, -1, -1, "", this.objectMapper.writeValueAsString(mtbFile), headers, Optional.empty())
)
verify(requestProcessor, times(1)).processDeletion(anyString(), anyString())
}
}

View File

@ -101,7 +101,7 @@ class KafkaMtbFileSenderTest {
val captor = argumentCaptor<String>()
verify(kafkaTemplate, times(1)).send(anyString(), captor.capture(), captor.capture())
assertThat(captor.firstValue).isNotNull
assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\", \"eid\": \"1\"}")
assertThat(captor.firstValue).isEqualTo("{\"pid\": \"PID\"}")
assertThat(captor.secondValue).isNotNull
assertThat(captor.secondValue).isEqualTo(objectMapper.writeValueAsString(kafkaRecordData("TestID", Consent.Status.ACTIVE)))
}

View File

@ -20,9 +20,10 @@
package dev.dnpm.etl.processor.pseudonym
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.MtbFile
import de.ukw.ccc.bwhc.dto.*
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.assertThrows
import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.ArgumentMatchers
import org.mockito.Mock
@ -61,4 +62,137 @@ class ExtensionsTest {
assertThat(mtbFile.serialized()).doesNotContain(CLEAN_PATIENT_ID)
}
@Test
fun shouldNotContainAnyUuidAfterRehashingOfIds(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(ArgumentMatchers.anyString())
doAnswer {
"TESTDOMAIN"
}.whenever(pseudonymizeService).prefix()
val mtbFile = fakeMtbFile()
mtbFile.pseudonymizeWith(pseudonymizeService)
mtbFile.anonymizeContentWith(pseudonymizeService)
val pattern = "\"[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\"".toRegex().toPattern()
val matcher = pattern.matcher(mtbFile.serialized())
assertThrows<IllegalStateException> {
matcher.find()
matcher.group()
}.also {
assertThat(it.message).isEqualTo("No match found")
}
}
@Test
fun shouldRehashIdsWithPrefix(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(ArgumentMatchers.anyString())
doAnswer {
"TESTDOMAIN"
}.whenever(pseudonymizeService).prefix()
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.build()
mtbFile.pseudonymizeWith(pseudonymizeService)
mtbFile.anonymizeContentWith(pseudonymizeService)
assertThat(mtbFile.episode.id)
// TESTDOMAIN<sha256(TESTDOMAIN-1)[0-41]>
.isEqualTo("TESTDOMAIN44e20a53bbbf9f3ae39626d05df7014dcd77d6098")
}
@Test
fun shouldNotThrowExceptionOnNullValues(@Mock pseudonymizeService: PseudonymizeService) {
doAnswer {
it.arguments[0]
"PSEUDO-ID"
}.whenever(pseudonymizeService).patientPseudonym(ArgumentMatchers.anyString())
doAnswer {
"TESTDOMAIN"
}.whenever(pseudonymizeService).prefix()
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.withClaims(null)
.withDiagnoses(null)
.withCarePlans(null)
.withClaimResponses(null)
.withEcogStatus(null)
.withFamilyMemberDiagnoses(null)
.withGeneticCounsellingRequests(null)
.withHistologyReevaluationRequests(null)
.withHistologyReports(null)
.withLastGuidelineTherapies(null)
.withMolecularPathologyFindings(null)
.withMolecularTherapies(null)
.withNgsReports(null)
.withPreviousGuidelineTherapies(null)
.withRebiopsyRequests(null)
.withRecommendations(null)
.withResponses(null)
.withStudyInclusionRequests(null)
.withSpecimens(null)
.build()
mtbFile.pseudonymizeWith(pseudonymizeService)
mtbFile.anonymizeContentWith(pseudonymizeService)
assertThat(mtbFile.episode.id).isNotNull()
}
}

View File

@ -43,20 +43,23 @@ class ReportServiceTest {
"issues": [
{ "severity": "info", "message": "Info Message" },
{ "severity": "warning", "message": "Warning Message" },
{ "severity": "error", "message": "Error Message" }
{ "severity": "error", "message": "Error Message" },
{ "severity": "fatal", "message": "Fatal Message" }
]
}
""".trimIndent()
val actual = this.reportService.deserialize(json)
assertThat(actual).hasSize(3)
assertThat(actual[0].severity).isEqualTo(ReportService.Severity.ERROR)
assertThat(actual[0].message).isEqualTo("Error Message")
assertThat(actual[1].severity).isEqualTo(ReportService.Severity.WARNING)
assertThat(actual[1].message).isEqualTo("Warning Message")
assertThat(actual[2].severity).isEqualTo(ReportService.Severity.INFO)
assertThat(actual[2].message).isEqualTo("Info Message")
assertThat(actual).hasSize(4)
assertThat(actual[0].severity).isEqualTo(ReportService.Severity.FATAL)
assertThat(actual[0].message).isEqualTo("Fatal Message")
assertThat(actual[1].severity).isEqualTo(ReportService.Severity.ERROR)
assertThat(actual[1].message).isEqualTo("Error Message")
assertThat(actual[2].severity).isEqualTo(ReportService.Severity.WARNING)
assertThat(actual[2].message).isEqualTo("Warning Message")
assertThat(actual[3].severity).isEqualTo(ReportService.Severity.INFO)
assertThat(actual[3].message).isEqualTo("Info Message")
}
@Test

View File

@ -92,7 +92,7 @@ class RequestProcessorTest {
uuid = UUID.randomUUID().toString(),
patientId = "TEST_12345678901",
pid = "P1",
fingerprint = "xrysxpozhbs2lnrjgf3yq4fzj33kxr7xr5c2cbuskmelfdmckl3a",
fingerprint = "zdlzv5s5ydmd4ktw2v5piohegc4jcyrm6j66bq6tv2uxuerndmga",
type = RequestType.MTB_FILE,
status = RequestStatus.SUCCESS,
processedAt = Instant.parse("2023-08-08T02:00:00Z")
@ -151,7 +151,7 @@ class RequestProcessorTest {
uuid = UUID.randomUUID().toString(),
patientId = "TEST_12345678901",
pid = "P1",
fingerprint = "xrysxpozhbs2lnrjgf3yq4fzj33kxr7xr5c2cbuskmelfdmckl3a",
fingerprint = "zdlzv5s5ydmd4ktw2v5piohegc4jcyrm6j66bq6tv2uxuerndmga",
type = RequestType.MTB_FILE,
status = RequestStatus.SUCCESS,
processedAt = Instant.parse("2023-08-08T02:00:00Z")

File diff suppressed because one or more lines are too long