1
0
mirror of https://github.com/pcvolkmer/etl-processor.git synced 2025-07-01 14:12:55 +00:00

36 Commits

Author SHA1 Message Date
6806c4fd69 build: bump version 2024-03-04 13:21:07 +01:00
b2016df852 style: fix some style issued 2024-03-04 13:17:57 +01:00
b332f3c5ff Merge pull request #50 from CCC-MF/issue_49
Administrative Rechte für OIDC-Benutzer
2024-03-04 12:54:31 +01:00
9eb65788e1 style: change login/logout style 2024-03-04 12:50:07 +01:00
9392bcadc9 feat: add admin role assignment 2024-03-04 10:12:12 +01:00
a008641192 fix: remove maxSessionsPreventsLogin 2024-03-01 15:07:23 +01:00
5928d52237 Merge pull request #48 from CCC-MF/issue_36
Freigabe und Berechtigung für OIDC-Benutzer
2024-03-01 14:09:06 +01:00
1eb40b40c9 docs: add documentation for user roles 2024-03-01 14:02:50 +01:00
feb9f2430c feat: add config page for user role assignment 2024-03-01 13:51:06 +01:00
200c5338ea feat: add default new user role config option 2024-03-01 09:34:51 +01:00
5c15ad4518 feat: add user role database table and role-based permissions 2024-03-01 09:30:07 +01:00
0b6decf88d Merge pull request #47 from CCC-MF/issue_43
feat: add config option to deactivate duplication check
2024-03-01 07:33:16 +01:00
cfdf41d550 feat: add config option to deactivate duplication check 2024-03-01 07:27:58 +01:00
45c65d53ce docs: add information about Kafka input topics 2024-02-29 13:55:14 +01:00
4568f491f5 Merge pull request #46 from CCC-MF/issue_42_kafka
Dateneingang über Apache-Kafka als Alternative zu HTTP-Request
2024-02-29 13:15:57 +01:00
952ad8c0cf test: add test for incoming kafka message processing 2024-02-29 12:49:06 +01:00
3e45bf8494 feat: implement KafkaInputListener 2024-02-29 09:19:32 +01:00
46ddaf10f7 Merge pull request #45 from CCC-MF/issue_34
Verwendung einer applikationsweiten Retry-Konfiguration
2024-02-29 08:57:00 +01:00
408b121f26 test: add test for max retry attempts 2024-02-29 08:47:17 +01:00
61e5273158 feat: add max-retry-attempts config option 2024-02-29 08:29:26 +01:00
50b8f7bbd4 feat: use global RetryTemplate 2024-02-29 08:26:54 +01:00
25f286f73b chore: update spring boot to version 3.2.3 2024-02-25 12:28:34 +01:00
50a6d66718 feat: new kafka config due to kafka input 2024-02-19 17:06:02 +01:00
f5c80f6d81 Merge pull request #41 from CCC-MF/issue_39
feat: add cache-control headers for static resources
2024-02-19 09:08:19 +01:00
7659939d3c Merge pull request #40 from CCC-MF/issue_37
build: use JDK 21 and update gradle version
2024-02-19 09:02:40 +01:00
f58d4a76cf build: use JDK 21 within workflows 2024-02-19 08:59:42 +01:00
c2dd450579 feat: add cache-control headers for static resources 2024-02-19 08:54:38 +01:00
a1b62ad754 build: use JDK 21 and update gradle version 2024-02-19 08:53:21 +01:00
59d8744c84 refactor: move mtb file controller into package input 2024-02-17 14:58:24 +01:00
d2a6ec17ea docs: add ports to example docker compose file 2024-02-16 13:00:22 +01:00
550403cc9f build: bump snapshot version 2024-02-15 13:41:11 +01:00
d3a4500568 Merge pull request #35 from CCC-MF/issue_33
Deprecate usage of ...SSL_CA_LOCATION config param
2024-02-09 08:19:07 +01:00
2e4fee97a8 feat: Deprecate usage of ...SSL_CA_LOCATION config param 2024-02-09 08:15:01 +01:00
5355eee05c docs: mark gPAS ssl config param as deprecated 2024-02-08 10:05:06 +01:00
3e22000541 Merge pull request #32 from CCC-MF/issue_31
build: update used actions
2024-02-05 08:18:26 +01:00
8c319197d0 build: update used actions 2024-02-05 08:12:28 +01:00
36 changed files with 821 additions and 168 deletions

View File

@ -1,4 +1,4 @@
name: "Run build and deploy"
name: 'Run build and deploy'
on:
release:
@ -8,17 +8,17 @@ jobs:
docker:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
java-version: '17'
java-version: '21'
distribution: 'temurin'
- name: Setup Gradle
uses: gradle/gradle-build-action@v2.4.2
uses: gradle/actions/setup-gradle@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v2

View File

@ -11,14 +11,14 @@ jobs:
tests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
java-version: '17'
java-version: '21'
distribution: 'temurin'
- name: Setup Gradle
uses: gradle/gradle-build-action@v2.4.2
uses: gradle/actions/setup-gradle@v3
- name: Execute tests
run: ./gradlew test
@ -26,14 +26,14 @@ jobs:
integrationTests:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-java@v3
with:
java-version: '17'
java-version: '21'
distribution: 'temurin'
- name: Setup Gradle
uses: gradle/gradle-build-action@v2.4.2
uses: gradle/actions/setup-gradle@v3
- name: Execute integration tests
run: ./gradlew integrationTest

View File

@ -15,6 +15,11 @@ Zudem ist eine minimalistische Weboberfläche integriert, die einen Einblick in
![Modell DNPM-ETL-Strecke](docs/etl.png)
### Duplikaterkennung
Die Erkennung von Duplikaten ist normalerweise immer aktiv, kann jedoch über den Konfigurationsparameter
`APP_DUPLICATION_DETECTION=false` deaktiviert werden.
### Datenübermittlung über HTTP/REST
Anfragen werden, wenn nicht als Duplikat behandelt, nach der Pseudonymisierung direkt an das bwHC-Backend gesendet.
@ -59,7 +64,10 @@ Wurde die Verwendung von gPAS konfiguriert, so sind weitere Angaben zu konfiguri
* `APP_PSEUDONYMIZE_GPAS_TARGET`: gPas Domänenname
* `APP_PSEUDONYMIZE_GPAS_USERNAME`: gPas Basic-Auth Benutzername
* `APP_PSEUDONYMIZE_GPAS_PASSWORD`: gPas Basic-Auth Passwort
* `APP_PSEUDONYMIZE_GPAS_SSLCALOCATION`: Root Zertifikat für gPas, falls es dediziert hinzugefügt werden muss.
* ~~`APP_PSEUDONYMIZE_GPAS_SSLCALOCATION`~~: **Veraltet** - Root Zertifikat für gPas, falls es dediziert hinzugefügt werden muss.
Der Konfigurationsparameter `APP_PSEUDONYMIZE_GPAS_SSLCALOCATION` sollte nicht mehr verwendet werden und wird in einer kommenden Version entfernt.
Stattdessen sollte das Root Zertifikat wie unter [_Integration eines eigenen Root CA Zertifikats_](#integration-eines-eigenen-root-ca-zertifikats) beschrieben eingebunden werden.
### Anmeldung mit einem Passwort
@ -106,6 +114,23 @@ Weitere Informationen zur Konfiguration des OIDC-Providers
sind [hier](https://docs.spring.io/spring-security/reference/servlet/oauth2/index.html#oauth2-client)
zu finden.
#### Rollenbasierte Berechtigungen
Wird OpenID Connect verwendet, gibt es eine rollenbasierte Berechtigungszuweisung.
Die Standardrolle für neue OIDC-Benutzer kann mit der Option `APP_SECURITY_DEFAULT_USER_ROLE` festgelegt werden.
Mögliche Werte sind `user` oder `guest`. Standardwert ist `user`.
Benutzer mit der Rolle "Gast" sehen nur die Inhalte, die auch nicht angemeldete Benutzer sehen.
Hierdurch ist es möglich, einzelne Benutzer einzuschränken oder durch Änderung der Standardrolle auf `guest` nur
einzelne Benutzer als vollwertige Nutzer zuzulassen.
![Rollenverwaltung](docs/userroles.png)
Benutzer werden nach dem Entfernen oder der Änderung der vergebenen Rolle automatisch abgemeldet und müssen sich neu anmelden.
Sie bekommen dabei wieder die Standardrolle zugewiesen.
#### Auswirkungen auf den dargestellten Inhalt
Nur Administratoren haben Zugriff auf den Konfigurationsbereich, nur angemeldete Benutzer können die anonymisierte oder
@ -164,8 +189,10 @@ Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an das
Folgende Umgebungsvariablen müssen gesetzt sein, damit ein bwHC-MTB-File an ein Kafka-Topic übermittelt wird:
* `APP_KAFKA_TOPIC`: Zu verwendendes Topic zum Versenden von Anfragen
* `APP_KAFKA_RESPONSE_TOPIC`: Topic mit Antworten über den Erfolg des Versendens. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_response".
* `APP_KAFKA_OUTPUT_TOPIC`: Zu verwendendes Topic zum Versenden von Anfragen.
Ersetzt in einer kommenden Version `APP_KAFKA_TOPIC`.
* `APP_KAFKA_OUTPUT_RESPONSE_TOPIC`: Topic mit Antworten über den Erfolg des Versendens. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_response".
Ersetzt in einer kommenden Version `APP_KAFKA_RESPONSE_TOPIC`.
* `APP_KAFKA_GROUP_ID`: Kafka GroupID des Consumers. Standardwert: `APP_KAFKA_TOPIC` mit Anhang "_group".
* `APP_KAFKA_SERVERS`: Zu verwendende Kafka-Bootstrap-Server als kommagetrennte Liste
@ -176,6 +203,8 @@ Weitere Einstellungen können über die Parameter von Spring Kafka konfiguriert
Lässt sich keine Verbindung zu dem bwHC-Backend aufbauen, wird eine Rückantwort mit Status-Code `900` erwartet, welchen es
für HTTP nicht gibt.
Wird die Umgebungsvariable `APP_KAFKA_INPUT_TOPIC` gesetzt, kann eine Nachricht auch über dieses Kafka-Topic an den ETL-Prozessor übermittelt werden.
##### Retention Time
Generell werden in Apache Kafka alle Records entsprechend der Konfiguration vorgehalten.

View File

@ -4,14 +4,14 @@ import org.springframework.boot.gradle.tasks.bundling.BootBuildImage
plugins {
war
id("org.springframework.boot") version "3.2.2"
id("org.springframework.boot") version "3.2.3"
id("io.spring.dependency-management") version "1.1.4"
kotlin("jvm") version "1.9.22"
kotlin("plugin.spring") version "1.9.22"
}
group = "de.ukw.ccc"
version = "0.7.0"
version = "0.8.0"
var versions = mapOf(
"bwhc-dto-java" to "0.2.0",
@ -24,7 +24,7 @@ var versions = mapOf(
)
java {
sourceCompatibility = JavaVersion.VERSION_17
sourceCompatibility = JavaVersion.VERSION_21
}
sourceSets {
@ -90,7 +90,7 @@ dependencies {
tasks.withType<KotlinCompile> {
kotlinOptions {
freeCompilerArgs += "-Xjsr305=strict"
jvmTarget = "17"
jvmTarget = "21"
}
}

View File

@ -9,6 +9,8 @@ services:
### ETL-Processor
etl-processor:
image: ghcr.io/ccc-mf/etl-processor:latest
ports:
- 8080:8080
environment:
APP_REST_URI: http://bwhc-backend/bwhc/etl/api
SPRING_DATASOURCE_URL: jdbc:postgresql://postgres/etl

BIN
docs/userroles.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

View File

@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-bin.zip
networkTimeout=10000
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

View File

@ -20,11 +20,13 @@
package dev.dnpm.etl.processor.config
import com.fasterxml.jackson.databind.ObjectMapper
import dev.dnpm.etl.processor.input.KafkaInputListener
import dev.dnpm.etl.processor.monitoring.RequestRepository
import dev.dnpm.etl.processor.output.KafkaMtbFileSender
import dev.dnpm.etl.processor.output.RestMtbFileSender
import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator
import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.dnpm.etl.processor.services.TokenRepository
import dev.dnpm.etl.processor.services.TokenService
import org.assertj.core.api.Assertions.assertThat
@ -37,6 +39,7 @@ import org.springframework.boot.test.context.SpringBootTest
import org.springframework.boot.test.mock.mockito.MockBean
import org.springframework.boot.test.mock.mockito.MockBeans
import org.springframework.context.ApplicationContext
import org.springframework.retry.support.RetryTemplate
import org.springframework.security.crypto.password.PasswordEncoder
import org.springframework.security.provisioning.InMemoryUserDetailsManager
import org.springframework.test.context.ContextConfiguration
@ -78,8 +81,8 @@ class AppConfigurationTest {
@TestPropertySource(
properties = [
"app.kafka.servers=localhost:9092",
"app.kafka.topic=test",
"app.kafka.response-topic=test-response",
"app.kafka.output-topic=test",
"app.kafka.output-response-topic=test-response",
"app.kafka.group-id=test"
]
)
@ -99,8 +102,8 @@ class AppConfigurationTest {
properties = [
"app.rest.uri=http://localhost:9000",
"app.kafka.servers=localhost:9092",
"app.kafka.topic=test",
"app.kafka.response-topic=test-response",
"app.kafka.output-topic=test",
"app.kafka.output-response-topic=test-response",
"app.kafka.group-id=test"
]
)
@ -114,6 +117,44 @@ class AppConfigurationTest {
}
@Nested
@TestPropertySource(
properties = [
"app.kafka.servers=localhost:9092",
"app.kafka.output-topic=test",
"app.kafka.output-response-topic=test-response",
"app.kafka.group-id=test"
]
)
inner class AppConfigurationWithoutKafkaInputTest(private val context: ApplicationContext) {
@Test
fun shouldNotUseKafkaInputListener() {
assertThrows<NoSuchBeanDefinitionException> { context.getBean(KafkaInputListener::class.java) }
}
}
@Nested
@TestPropertySource(
properties = [
"app.kafka.servers=localhost:9092",
"app.kafka.input-topic=test_input",
"app.kafka.output-topic=test",
"app.kafka.output-response-topic=test-response",
"app.kafka.group-id=test"
]
)
@MockBean(RequestProcessor::class)
inner class AppConfigurationUsingKafkaInputTest(private val context: ApplicationContext) {
@Test
fun shouldUseKafkaInputListener() {
assertThat(context.getBean(KafkaInputListener::class.java)).isNotNull
}
}
@Nested
@TestPropertySource(
properties = [
@ -238,4 +279,30 @@ class AppConfigurationTest {
}
@Nested
@TestPropertySource(
properties = [
"app.rest.uri=http://localhost:9000",
"app.max-retry-attempts=5"
]
)
inner class AppConfigurationRetryTest(private val context: ApplicationContext) {
private val maxRetryAttempts = 5
@Test
fun shouldUseRetryTemplateWithConfiguredMaxAttempts() {
val retryTemplate = context.getBean(RetryTemplate::class.java)
assertThat(retryTemplate).isNotNull
assertThrows<RuntimeException> {
retryTemplate.execute<Void, RuntimeException> {
assertThat(it.retryCount).isLessThan(maxRetryAttempts)
throw RuntimeException()
}
}
}
}
}

View File

@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.web
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -22,21 +22,6 @@ package dev.dnpm.etl.processor.pseudonym;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.parser.IParser;
import dev.dnpm.etl.processor.config.GPasConfigProperties;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.ConnectException;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.util.Base64;
import java.util.HashMap;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
@ -54,35 +39,39 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.*;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.retry.RetryCallback;
import org.springframework.retry.RetryContext;
import org.springframework.retry.RetryListener;
import org.springframework.retry.RetryPolicy;
import org.springframework.retry.backoff.ExponentialBackOffPolicy;
import org.springframework.retry.policy.SimpleRetryPolicy;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.util.Base64;
public class GpasPseudonymGenerator implements Generator {
private final static FhirContext r4Context = FhirContext.forR4();
private final String gPasUrl;
private final String psnTargetDomain;
private final HttpHeaders httpHeader;
private final RetryTemplate retryTemplate = defaultTemplate();
private final RetryTemplate retryTemplate;
private final Logger log = LoggerFactory.getLogger(GpasPseudonymGenerator.class);
private SSLContext customSslContext;
private RestTemplate restTemplate;
public GpasPseudonymGenerator(GPasConfigProperties gpasCfg) {
public GpasPseudonymGenerator(GPasConfigProperties gpasCfg, RetryTemplate retryTemplate) {
this.retryTemplate = retryTemplate;
this.gPasUrl = gpasCfg.getUri();
this.psnTargetDomain = gpasCfg.getTarget();
@ -91,7 +80,7 @@ public class GpasPseudonymGenerator implements Generator {
try {
if (StringUtils.isNotBlank(gpasCfg.getSslCaLocation())) {
customSslContext = getSslContext(gpasCfg.getSslCaLocation());
log.debug(String.format("%s has been initialized with SSL certificate %s",
log.warn(String.format("%s has been initialized with SSL certificate %s. This is deprecated in favor of including Root CA.",
this.getClass().getName(), gpasCfg.getSslCaLocation()));
}
} catch (IOException | KeyManagementException | KeyStoreException | CertificateException |
@ -202,31 +191,6 @@ public class GpasPseudonymGenerator implements Generator {
return headers;
}
protected RetryTemplate defaultTemplate() {
RetryTemplate retryTemplate = new RetryTemplate();
ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
backOffPolicy.setInitialInterval(1000);
backOffPolicy.setMultiplier(1.25);
retryTemplate.setBackOffPolicy(backOffPolicy);
HashMap<Class<? extends Throwable>, Boolean> retryableExceptions = new HashMap<>();
retryableExceptions.put(RestClientException.class, true);
retryableExceptions.put(ConnectException.class, true);
RetryPolicy retryPolicy = new SimpleRetryPolicy(3, retryableExceptions);
retryTemplate.setRetryPolicy(retryPolicy);
retryTemplate.registerListener(new RetryListener() {
@Override
public <T, E extends Throwable> void onError(RetryContext context,
RetryCallback<T, E> callback, Throwable throwable) {
log.warn("HTTP Error occurred: {}. Retrying {}", throwable.getMessage(),
context.getRetryCount());
RetryListener.super.onError(context, callback, throwable);
}
});
return retryTemplate;
}
/**
* Read SSL root certificate and return SSLContext
*

View File

@ -19,6 +19,7 @@
package dev.dnpm.etl.processor.config
import dev.dnpm.etl.processor.security.Role
import org.springframework.boot.context.properties.ConfigurationProperties
import org.springframework.boot.context.properties.DeprecatedConfigurationProperty
@ -30,7 +31,9 @@ data class AppConfigProperties(
replacement = "app.pseudonymize.generator"
)
var pseudonymizer: PseudonymGenerator = PseudonymGenerator.BUILDIN,
var transformations: List<TransformationProperties> = listOf()
var transformations: List<TransformationProperties> = listOf(),
var maxRetryAttempts: Int = 3,
var duplicationDetection: Boolean = true
) {
companion object {
const val NAME = "app"
@ -53,8 +56,10 @@ data class GPasConfigProperties(
val target: String = "etl-processor",
val username: String?,
val password: String?,
val sslCaLocation: String?,
@get:DeprecatedConfigurationProperty(
reason = "Deprecated in favor of including Root CA"
)
val sslCaLocation: String?
) {
companion object {
const val NAME = "app.pseudonymize.gpas"
@ -70,10 +75,21 @@ data class RestTargetProperties(
}
}
@ConfigurationProperties(KafkaTargetProperties.NAME)
data class KafkaTargetProperties(
val topic: String = "etl-processor",
val responseTopic: String = "${topic}_response",
@ConfigurationProperties(KafkaProperties.NAME)
data class KafkaProperties(
val inputTopic: String?,
val outputTopic: String = "etl-processor",
@get:DeprecatedConfigurationProperty(
reason = "Deprecated",
replacement = "outputTopic"
)
val topic: String = outputTopic,
val outputResponseTopic: String = "${outputTopic}_response",
@get:DeprecatedConfigurationProperty(
reason = "Deprecated",
replacement = "outputResponseTopic"
)
val responseTopic: String = outputResponseTopic,
val groupId: String = "${topic}_group",
val servers: String = ""
) {
@ -87,7 +103,8 @@ data class SecurityConfigProperties(
val adminUser: String?,
val adminPassword: String?,
val enableTokens: Boolean = false,
val enableOidc: Boolean = false
val enableOidc: Boolean = false,
val defaultNewUserRole: Role = Role.USER
) {
companion object {
const val NAME = "app.security"

View File

@ -35,13 +35,15 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.retry.RetryCallback
import org.springframework.retry.RetryContext
import org.springframework.retry.RetryListener
import org.springframework.retry.policy.SimpleRetryPolicy
import org.springframework.retry.support.RetryTemplate
import org.springframework.retry.support.RetryTemplateBuilder
import org.springframework.scheduling.annotation.EnableScheduling
import org.springframework.security.crypto.password.PasswordEncoder
import org.springframework.security.provisioning.InMemoryUserDetailsManager
import org.springframework.security.provisioning.UserDetailsManager
import reactor.core.publisher.Sinks
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
@ -62,8 +64,8 @@ class AppConfiguration {
@ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "GPAS")
@Bean
fun gpasPseudonymGenerator(configProperties: GPasConfigProperties): Generator {
return GpasPseudonymGenerator(configProperties)
fun gpasPseudonymGenerator(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate): Generator {
return GpasPseudonymGenerator(configProperties, retryTemplate)
}
@ConditionalOnProperty(value = ["app.pseudonymize.generator"], havingValue = "BUILDIN", matchIfMissing = true)
@ -75,8 +77,8 @@ class AppConfiguration {
@ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "GPAS")
@ConditionalOnMissingBean
@Bean
fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties): Generator {
return GpasPseudonymGenerator(configProperties)
fun gpasPseudonymGeneratorOnDeprecatedProperty(configProperties: GPasConfigProperties, retryTemplate: RetryTemplate): Generator {
return GpasPseudonymGenerator(configProperties, retryTemplate)
}
@ConditionalOnProperty(value = ["app.pseudonymizer"], havingValue = "BUILDIN")
@ -111,11 +113,20 @@ class AppConfiguration {
}
@Bean
fun retryTemplate(): RetryTemplate {
fun retryTemplate(configProperties: AppConfigProperties): RetryTemplate {
return RetryTemplateBuilder()
.notRetryOn(IllegalArgumentException::class.java)
.fixedBackoff(5.seconds.toJavaDuration())
.customPolicy(SimpleRetryPolicy(3))
.exponentialBackoff(2.seconds.toJavaDuration(), 1.25, 5.seconds.toJavaDuration())
.customPolicy(SimpleRetryPolicy(configProperties.maxRetryAttempts))
.withListener(object : RetryListener {
override fun <T : Any, E : Throwable> onError(
context: RetryContext,
callback: RetryCallback<T, E>,
throwable: Throwable
) {
logger.warn("Error occured: {}. Retrying {}", throwable.message, context.retryCount)
}
})
.build()
}

View File

@ -20,10 +20,12 @@
package dev.dnpm.etl.processor.config
import com.fasterxml.jackson.databind.ObjectMapper
import dev.dnpm.etl.processor.input.KafkaInputListener
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.monitoring.KafkaConnectionCheckService
import dev.dnpm.etl.processor.output.KafkaMtbFileSender
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.services.RequestProcessor
import dev.dnpm.etl.processor.services.kafka.KafkaResponseProcessor
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
@ -42,9 +44,9 @@ import reactor.core.publisher.Sinks
@Configuration
@EnableConfigurationProperties(
value = [KafkaTargetProperties::class]
value = [KafkaProperties::class]
)
@ConditionalOnProperty(value = ["app.kafka.topic", "app.kafka.servers"])
@ConditionalOnProperty(value = ["app.kafka.servers"])
@ConditionalOnMissingBean(MtbFileSender::class)
@Order(-5)
class AppKafkaConfiguration {
@ -54,21 +56,21 @@ class AppKafkaConfiguration {
@Bean
fun kafkaMtbFileSender(
kafkaTemplate: KafkaTemplate<String, String>,
kafkaTargetProperties: KafkaTargetProperties,
kafkaProperties: KafkaProperties,
retryTemplate: RetryTemplate,
objectMapper: ObjectMapper
): MtbFileSender {
logger.info("Selected 'KafkaMtbFileSender'")
return KafkaMtbFileSender(kafkaTemplate, kafkaTargetProperties, retryTemplate, objectMapper)
return KafkaMtbFileSender(kafkaTemplate, kafkaProperties, retryTemplate, objectMapper)
}
@Bean
fun kafkaListenerContainer(
fun kafkaResponseListenerContainer(
consumerFactory: ConsumerFactory<String, String>,
kafkaTargetProperties: KafkaTargetProperties,
kafkaProperties: KafkaProperties,
kafkaResponseProcessor: KafkaResponseProcessor
): KafkaMessageListenerContainer<String, String> {
val containerProperties = ContainerProperties(kafkaTargetProperties.responseTopic)
val containerProperties = ContainerProperties(kafkaProperties.responseTopic)
containerProperties.messageListener = kafkaResponseProcessor
return KafkaMessageListenerContainer(consumerFactory, containerProperties)
}
@ -81,6 +83,27 @@ class AppKafkaConfiguration {
return KafkaResponseProcessor(applicationEventPublisher, objectMapper)
}
@Bean
@ConditionalOnProperty(value = ["app.kafka.input-topic"])
fun kafkaInputListenerContainer(
consumerFactory: ConsumerFactory<String, String>,
kafkaProperties: KafkaProperties,
kafkaInputListener: KafkaInputListener
): KafkaMessageListenerContainer<String, String> {
val containerProperties = ContainerProperties(kafkaProperties.inputTopic)
containerProperties.messageListener = kafkaInputListener
return KafkaMessageListenerContainer(consumerFactory, containerProperties)
}
@Bean
@ConditionalOnProperty(value = ["app.kafka.input-topic"])
fun kafkaInputListener(
requestProcessor: RequestProcessor,
objectMapper: ObjectMapper
): KafkaInputListener {
return KafkaInputListener(requestProcessor, objectMapper)
}
@Bean
fun connectionCheckService(consumerFactory: ConsumerFactory<String, String>, configsUpdateProducer: Sinks.Many<Boolean>): ConnectionCheckService {
return KafkaConnectionCheckService(consumerFactory.createConsumer(), configsUpdateProducer)

View File

@ -19,6 +19,9 @@
package dev.dnpm.etl.processor.config
import dev.dnpm.etl.processor.security.UserRole
import dev.dnpm.etl.processor.security.UserRoleRepository
import dev.dnpm.etl.processor.services.UserRoleService
import org.slf4j.LoggerFactory
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.boot.context.properties.EnableConfigurationProperties
@ -27,10 +30,15 @@ import org.springframework.context.annotation.Configuration
import org.springframework.security.config.annotation.web.builders.HttpSecurity
import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity
import org.springframework.security.config.annotation.web.invoke
import org.springframework.security.core.authority.SimpleGrantedAuthority
import org.springframework.security.core.authority.mapping.GrantedAuthoritiesMapper
import org.springframework.security.core.session.SessionRegistry
import org.springframework.security.core.session.SessionRegistryImpl
import org.springframework.security.core.userdetails.User
import org.springframework.security.core.userdetails.UserDetails
import org.springframework.security.crypto.factory.PasswordEncoderFactories
import org.springframework.security.crypto.password.PasswordEncoder
import org.springframework.security.oauth2.core.oidc.user.OidcUserAuthority
import org.springframework.security.provisioning.InMemoryUserDetailsManager
import org.springframework.security.web.SecurityFilterChain
import java.util.*
@ -77,12 +85,19 @@ class AppSecurityConfiguration(
@Bean
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "true")
fun filterChainOidc(http: HttpSecurity, passwordEncoder: PasswordEncoder): SecurityFilterChain {
fun filterChainOidc(http: HttpSecurity, passwordEncoder: PasswordEncoder, userRoleRepository: UserRoleRepository, sessionRegistry: SessionRegistry): SecurityFilterChain {
http {
authorizeRequests {
authorize("/configs/**", hasRole("ADMIN"))
authorize("/mtbfile/**", hasAnyRole("MTBFILE"))
authorize("/report/**", fullyAuthenticated)
authorize("/report/**", hasAnyRole("ADMIN", "USER"))
authorize("*.css", permitAll)
authorize("*.ico", permitAll)
authorize("*.jpeg", permitAll)
authorize("*.js", permitAll)
authorize("*.svg", permitAll)
authorize("*.css", permitAll)
authorize("/login/**", permitAll)
authorize(anyRequest, permitAll)
}
httpBasic {
@ -94,11 +109,38 @@ class AppSecurityConfiguration(
oauth2Login {
loginPage = "/login"
}
sessionManagement {
sessionConcurrency {
maximumSessions = 1
expiredUrl = "/login?expired"
}
sessionFixation {
newSession()
}
}
csrf { disable() }
}
return http.build()
}
@Bean
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "true")
fun grantedAuthoritiesMapper(userRoleRepository: UserRoleRepository, appSecurityConfigProperties: SecurityConfigProperties): GrantedAuthoritiesMapper {
return GrantedAuthoritiesMapper { grantedAuthority ->
grantedAuthority.filterIsInstance<OidcUserAuthority>()
.onEach {
val userRole = userRoleRepository.findByUsername(it.userInfo.preferredUsername)
if (userRole.isEmpty) {
userRoleRepository.save(UserRole(null, it.userInfo.preferredUsername, appSecurityConfigProperties.defaultNewUserRole))
}
}
.map {
val userRole = userRoleRepository.findByUsername(it.userInfo.preferredUsername)
SimpleGrantedAuthority("ROLE_${userRole.get().role.toString().uppercase()}")
}
}
}
@Bean
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "false", matchIfMissing = true)
fun filterChain(http: HttpSecurity, passwordEncoder: PasswordEncoder): SecurityFilterChain {
@ -120,10 +162,19 @@ class AppSecurityConfiguration(
return http.build()
}
@Bean
fun sessionRegistry(): SessionRegistry {
return SessionRegistryImpl()
}
@Bean
fun passwordEncoder(): PasswordEncoder {
return PasswordEncoderFactories.createDelegatingPasswordEncoder()
}
@Bean
@ConditionalOnProperty(value = ["app.security.enable-oidc"], havingValue = "true")
fun userRoleService(userRoleRepository: UserRoleRepository, sessionRegistry: SessionRegistry): UserRoleService {
return UserRoleService(userRoleRepository, sessionRegistry)
}
}

View File

@ -0,0 +1,46 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.services.RequestProcessor
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.slf4j.LoggerFactory
import org.springframework.kafka.listener.MessageListener
class KafkaInputListener(
private val requestProcessor: RequestProcessor,
private val objectMapper: ObjectMapper
) : MessageListener<String, String> {
private val logger = LoggerFactory.getLogger(KafkaInputListener::class.java)
override fun onMessage(data: ConsumerRecord<String, String>) {
val mtbFile = objectMapper.readValue(data.value(), MtbFile::class.java)
if (mtbFile.consent.status == Consent.Status.ACTIVE) {
logger.debug("Accepted MTB File for processing")
requestProcessor.processMtbFile(mtbFile)
} else {
logger.debug("Accepted MTB File and process deletion")
requestProcessor.processDeletion(mtbFile.patient.id)
}
}
}

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.web
package dev.dnpm.etl.processor.input
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile

View File

@ -22,7 +22,7 @@ package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.config.KafkaTargetProperties
import dev.dnpm.etl.processor.config.KafkaProperties
import dev.dnpm.etl.processor.monitoring.RequestStatus
import org.slf4j.LoggerFactory
import org.springframework.kafka.core.KafkaTemplate
@ -30,7 +30,7 @@ import org.springframework.retry.support.RetryTemplate
class KafkaMtbFileSender(
private val kafkaTemplate: KafkaTemplate<String, String>,
private val kafkaTargetProperties: KafkaTargetProperties,
private val kafkaProperties: KafkaProperties,
private val retryTemplate: RetryTemplate,
private val objectMapper: ObjectMapper
) : MtbFileSender {
@ -41,7 +41,7 @@ class KafkaMtbFileSender(
return try {
return retryTemplate.execute<MtbFileSender.Response, Exception> {
val result = kafkaTemplate.send(
kafkaTargetProperties.topic,
kafkaProperties.topic,
key(request),
objectMapper.writeValueAsString(Data(request.requestId, request.mtbFile))
)
@ -71,7 +71,7 @@ class KafkaMtbFileSender(
return try {
return retryTemplate.execute<MtbFileSender.Response, Exception> {
val result = kafkaTemplate.send(
kafkaTargetProperties.topic,
kafkaProperties.topic,
key(request),
objectMapper.writeValueAsString(Data(request.requestId, dummyMtbFile))
)
@ -90,7 +90,7 @@ class KafkaMtbFileSender(
}
override fun endpoint(): String {
return "${this.kafkaTargetProperties.servers} (${this.kafkaTargetProperties.topic}/${this.kafkaTargetProperties.responseTopic})"
return "${this.kafkaProperties.servers} (${this.kafkaProperties.topic}/${this.kafkaProperties.responseTopic})"
}
private fun key(request: MtbFileSender.MtbFileRequest): String {

View File

@ -0,0 +1,45 @@
/*
* This file is part of ETL-Processor
*
* Copyright (C) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package dev.dnpm.etl.processor.security
import org.springframework.data.annotation.Id
import org.springframework.data.relational.core.mapping.Table
import org.springframework.data.repository.CrudRepository
import java.util.*
@Table("user_role")
data class UserRole(
@Id val id: Long? = null,
val username: String,
var role: Role = Role.GUEST
)
enum class Role(val value: String) {
GUEST("guest"),
USER("user"),
ADMIN("admin")
}
interface UserRoleRepository : CrudRepository<UserRole, Long> {
fun findByUsername(username: String): Optional<UserRole>
}

View File

@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.MtbFile
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.monitoring.Report
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
@ -42,7 +43,8 @@ class RequestProcessor(
private val sender: MtbFileSender,
private val requestService: RequestService,
private val objectMapper: ObjectMapper,
private val applicationEventPublisher: ApplicationEventPublisher
private val applicationEventPublisher: ApplicationEventPublisher,
private val appConfigProperties: AppConfigProperties
) {
fun processMtbFile(mtbFile: MtbFile) {
@ -64,7 +66,7 @@ class RequestProcessor(
)
)
if (isDuplication(mtbFile)) {
if (appConfigProperties.duplicationDetection && isDuplication(mtbFile)) {
applicationEventPublisher.publishEvent(
ResponseEvent(
requestId,

View File

@ -0,0 +1,61 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.services
import dev.dnpm.etl.processor.security.Role
import dev.dnpm.etl.processor.security.UserRole
import dev.dnpm.etl.processor.security.UserRoleRepository
import org.springframework.data.repository.findByIdOrNull
import org.springframework.security.core.session.SessionRegistry
import org.springframework.security.oauth2.core.oidc.user.OidcUser
class UserRoleService(
private val userRoleRepository: UserRoleRepository,
private val sessionRegistry: SessionRegistry
) {
fun updateUserRole(id: Long, role: Role) {
val userRole = userRoleRepository.findByIdOrNull(id) ?: return
userRole.role = role
userRoleRepository.save(userRole)
expireSessionFor(userRole.username)
}
fun deleteUserRole(id: Long) {
val userRole = userRoleRepository.findByIdOrNull(id) ?: return
userRoleRepository.delete(userRole)
expireSessionFor(userRole.username)
}
fun findAll(): List<UserRole> {
return userRoleRepository.findAll().toList()
}
private fun expireSessionFor(username: String) {
sessionRegistry.allPrincipals
.filterIsInstance<OidcUser>()
.filter { it.preferredUsername == username }
.flatMap {
sessionRegistry.getAllSessions(it, true)
}
.onEach {
it.expireNow()
}
}
}

View File

@ -22,9 +22,12 @@ package dev.dnpm.etl.processor.web
import dev.dnpm.etl.processor.monitoring.ConnectionCheckService
import dev.dnpm.etl.processor.output.MtbFileSender
import dev.dnpm.etl.processor.pseudonym.Generator
import dev.dnpm.etl.processor.security.Role
import dev.dnpm.etl.processor.security.UserRole
import dev.dnpm.etl.processor.services.Token
import dev.dnpm.etl.processor.services.TokenService
import dev.dnpm.etl.processor.services.TransformationService
import dev.dnpm.etl.processor.services.UserRoleService
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.http.MediaType
import org.springframework.http.codec.ServerSentEvent
@ -43,7 +46,8 @@ class ConfigController(
private val pseudonymGenerator: Generator,
private val mtbFileSender: MtbFileSender,
private val connectionCheckService: ConnectionCheckService,
private val tokenService: TokenService?
private val tokenService: TokenService?,
private val userRoleService: UserRoleService?
) {
@GetMapping
@ -56,10 +60,16 @@ class ConfigController(
if (tokenService != null) {
model.addAttribute("tokens", tokenService.findAll())
} else {
model.addAttribute("tokens", listOf<Token>())
model.addAttribute("tokens", emptyList<Token>())
}
model.addAttribute("transformations", transformationService.getTransformations())
if (userRoleService != null) {
model.addAttribute("userRolesEnabled", true)
model.addAttribute("userRoles", userRoleService.findAll())
} else {
model.addAttribute("userRolesEnabled", false)
model.addAttribute("userRoles", emptyList<UserRole>())
}
return "configs"
}
@ -112,6 +122,34 @@ class ConfigController(
return "configs/tokens"
}
@DeleteMapping(path = ["userroles/{id}"])
fun deleteUserRole(@PathVariable id: Long, model: Model): String {
if (userRoleService != null) {
userRoleService.deleteUserRole(id)
model.addAttribute("userRolesEnabled", true)
model.addAttribute("userRoles", userRoleService.findAll())
} else {
model.addAttribute("userRolesEnabled", false)
model.addAttribute("userRoles", emptyList<UserRole>())
}
return "configs/userroles"
}
@PutMapping(path = ["userroles/{id}"])
fun updateUserRole(@PathVariable id: Long, @ModelAttribute("role") role: Role, model: Model): String {
if (userRoleService != null) {
userRoleService.updateUserRole(id, role)
model.addAttribute("userRolesEnabled", true)
model.addAttribute("userRoles", userRoleService.findAll())
} else {
model.addAttribute("userRolesEnabled", false)
model.addAttribute("userRoles", emptyList<UserRole>())
}
return "configs/userroles"
}
@GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE])
fun events(): Flux<ServerSentEvent<Any>> {
return configsUpdateProducer.asFlux().map {

View File

@ -6,5 +6,16 @@ spring:
flyway:
locations: "classpath:db/migration/{vendor}"
web:
resources:
cache:
cachecontrol:
max-age: 1d
chain:
strategy:
content:
enabled: true
paths: /**/*.js,/**/*.css,/**/*.svg,/**/*.jpeg
server:
forward-headers-strategy: framework

View File

@ -0,0 +1,7 @@
CREATE TABLE IF NOT EXISTS user_role
(
id int auto_increment primary key,
username varchar(255) not null unique,
role varchar(255) not null,
created_at datetime default utc_timestamp() not null
);

View File

@ -0,0 +1,8 @@
CREATE TABLE IF NOT EXISTS user_role
(
id serial,
username varchar(255) not null unique,
role varchar(255) not null,
created_at timestamp with time zone default now() not null,
PRIMARY KEY (id)
);

View File

@ -2,6 +2,8 @@
--text: #333;
--table-border: rgba(16, 24, 40, .1);
--dark: brightness(.90);
--bg-blue: rgb(0, 74, 157);
--bg-blue-op: rgba(0, 74, 157, .35);
@ -40,7 +42,7 @@ body {
nav {
margin: 0 auto;
padding: 2em 0;
padding: 1em 0;
line-height: 1.5rem;
max-width: 1140px;
@ -48,18 +50,18 @@ nav {
border-bottom: 1px solid var(--table-border);
}
nav > a.nav-home {
nav a.nav-home {
float: left;
color: var(--text);
line-height: 1.5em;
text-decoration: none;
font-size: 1.5em;
font-size: 2em;
font-weight: bold;
}
nav > a.nav-home > img {
nav a.nav-home > img {
width: 1.5em;
vertical-align: middle;
}
@ -80,6 +82,57 @@ nav > ul > li.login {
margin: 0 0 0 1em;
padding: 0 0 0 2em;
border-left: 1px solid var(--table-border);
line-height: 3.5em;
}
nav > ul > li.login a {
text-decoration: none;
text-transform: none;
padding: 1em;
}
nav .login .user-name {
font-weight: bold;
}
nav > ul > li.login > span {
display: inline-block;
margin: 0 .5em;
}
nav > ul > li.login .user-icon {
flex-direction: column;
display: inline flex;
vertical-align: middle;
inline-size: 4em;
}
nav > ul > li.login .user-icon img {
margin: 0 0 -1em 0;
width: 80%;
align-self: center;
}
nav > ul > li.login .user-icon span {
padding: 0 .6em;
color: white;
font-size: .8em;
font-weight: bold;
border-radius: 4px;
line-height: normal;
text-align: center;
}
nav > ul > li.login .user-icon span.guest {
background: darkslategray;
}
nav > ul > li.login .user-icon span.user {
background: darkgreen;
}
nav > ul > li.login .user-icon span.admin {
background: darkred;
}
nav li a {
@ -89,10 +142,6 @@ nav li a {
font-weight: 700;
}
nav li.login a {
color: var(--bg-red);
}
nav li a:hover {
text-decoration: underline;
}
@ -202,6 +251,23 @@ form.samplecode-input input:focus-visible {
background: none;
}
.login-form img {
margin: 0 auto;
width: 4em;
display: block;
}
.userrole-form form {
margin: 0;
padding: 0;
border: none;
border-radius: 0;
background: none;
text-align: inherit;
}
.login-form form *,
.token-form form * {
padding: 0.5em;
@ -210,7 +276,8 @@ form.samplecode-input input:focus-visible {
}
.login-form form hr,
.token-form form hr {
.token-form form hr,
.userrole-form form hr {
padding: 0;
width: 100%;
}
@ -224,6 +291,14 @@ form.samplecode-input input:focus-visible {
border: none;
}
.userrole-form form select {
padding: 0.5em;
border: none;
border-radius: 3px;
line-height: 1.2rem;
font-size: 0.8rem;
}
.border {
padding: 1.5em;
border: 1px solid var(--table-border);
@ -402,7 +477,6 @@ td.clipboard.clipped {
padding: 4px 8px;
line-height: 1.2rem;
vertical-align: middle;
border: 0 solid transparent;
border-radius: 3px;
@ -414,38 +488,38 @@ td.clipboard.clipped {
cursor: pointer;
}
.btn:active,
.btn:hover {
filter: drop-shadow(1px 2px 2px gray);
filter: drop-shadow(1px 1px 1px gray) var(--dark);
}
.btn:active {
filter: drop-shadow(1px 1px 2px gray);
translate: 0 1px;
}
.btn.btn-red {
background: red;
background: var(--bg-red);
color: white;
}
.btn.btn-red:hover, .btn.btn-red:active {
background: darkred !important;
}
.btn.btn-blue {
background: slategray;
background: var(--bg-blue);
color: white;
}
.btn.btn-blue:hover, .btn.btn-blue:active {
background: darkslategray !important;
}
.btn.btn-delete:before {
content: '\1F5D1';
padding: .2rem;
}
button:disabled,
.btn:disabled {
background: slategray !important;
color: lightgray;
filter: none;
cursor: default;
}
input.inline {
border: none;
font-size: 1.1rem;
@ -527,6 +601,10 @@ input.inline:focus-visible {
color: var(--bg-green);
}
.notification.notice {
color: var(--bg-yellow);
}
.notification.error {
color: var(--bg-red);
}

View File

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg width="24" height="24" version="1.1" viewBox="0 0 6.35 6.35" xmlns="http://www.w3.org/2000/svg">
<g transform="matrix(1.2 0 0 1.2 -108.01 -85.977)">
<rect x="90.01" y="71.647" width="5.2917" height="5.2917" rx=".96212" fill="#b3b3b3"/>
<g transform="matrix(1.6667 0 0 1.6667 -60.888 -47.952)" fill="#fff">
<circle cx="92.126" cy="72.802" r=".70556"/>
<path d="m91.068 74.598a1.0583 1.0583 0 0 1 1.0583-1.0583 1.0583 1.0583 0 0 1 1.0583 1.0583h-1.0583z"/>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 588 B

View File

@ -40,6 +40,9 @@
<section th:insert="~{configs/tokens.html}">
</section>
<section th:insert="~{configs/userroles.html}">
</section>
<section hx-ext="sse" th:sse-connect="@{/configs/events}">
<div th:insert="~{configs/connectionAvailable.html}" th:hx-get="@{/configs?connectionAvailable}" hx-trigger="sse:connection-available">
</div>

View File

@ -32,7 +32,7 @@
<div class="token-form">
<form th:hx-post="@{/configs/tokens}" hx-target="#tokens">
<input placeholder="Token-Name" name="name" required />
<button>Token Erstellen</button>
<button class="btn">Token Erstellen</button>
</form>
</div>
</div>

View File

@ -0,0 +1,40 @@
<div th:if="${not userRolesEnabled}">
<h2><span></span> Benutzerberechtigungen</h2>
<p>Die Verwendung von rollenbasierten Benutzerberechtigungen ist nicht aktiviert.</p>
</div>
<div id="userroles" th:if="${userRolesEnabled}">
<h2><span></span> Benutzerberechtigungen</h2>
<div class="border">
<div th:if="${userRoles.isEmpty()}">Noch keine Benutzerberechtigungen vorhanden.</div>
<table th:if="${not userRoles.isEmpty()}">
<thead>
<tr>
<th>Benutzername</th>
<th>Rolle</th>
<th></th>
</tr>
</thead>
<tbody>
<tr th:each="userRole : ${userRoles}">
<td>[[ ${userRole.username} ]]</td>
<td>
<div class="userrole-form">
<form th:hx-put="@{/configs/userroles/{id}(id=${userRole.id})}" hx-target="#userroles">
<select name="role" th:disabled="${#authorization.authentication.getName() == userRole.username}">
<option th:selected="${userRole.role.value == 'guest'}" value="GUEST">Gast</option>
<option th:selected="${userRole.role.value == 'user'}" value="USER">Benutzer</option>
<option th:selected="${userRole.role.value == 'admin'}" value="ADMIN">Administrator</option>
</select>
<button class="btn btn-blue" th:disabled="${#authorization.authentication.getName() == userRole.username}">Übernehmen</button>
</form>
</div>
</td>
<td>
<button class="btn btn-red" th:hx-delete="@{/configs/userroles/{id}(id=${userRole.id})}" hx-target="#userroles" th:disabled="${#authorization.authentication.getName() == userRole.username}">Löschen</button>
</td>
</tr>
</tbody>
</table>
</div>
</div>

View File

@ -7,10 +7,12 @@
<body>
<div th:fragment="nav">
<nav>
<span>
<a class="nav-home" th:href="@{/}">
<img th:src="@{/icon.svg}" alt="Icon" />
<span>ETL-Processor</span>
</a>
</span>
<ul>
<li><a th:href="@{/}">Übersicht</a></li>
<li><a th:href="@{/statistics}">Statistiken</a></li>
@ -18,15 +20,19 @@
<a th:href="@{/configs}">Konfiguration</a>
</li>
<li class="login" sec:authorize="not isAuthenticated()">
<a th:href="@{/login}">Login</a>
<a class="btn btn-blue" th:href="@{/login}">Login</a>
</li>
<li class="login" sec:authorize="isAuthenticated()">
<span>
<span>👤</span>
<span sec:authentication="name">?</span>
<div class="user-icon">
<img th:src="@{/user.svg}" alt="User-Image">
<span sec:authorize="hasRole('ADMIN')" class="user-role admin">Admin</span>
<span sec:authorize="hasRole('USER')" class="user-role user">User</span>
<span sec:authorize="hasRole('GUEST')" class="user-role guest">Guest</span>
</div>
<span class="user-name" sec:authentication="name">?</span>
</span>
&nbsp;
<a th:href="@{/logout}">Abmelden</a>
<a class="btn btn-red" th:href="@{/logout}">Abmelden</a>
</li>
</ul>
</nav>

View File

@ -53,17 +53,17 @@
<td th:style="${request.type.value == 'delete'} ? 'color: red;'"><small>[[ ${request.type} ]]</small></td>
<td th:if="not ${request.report}">[[ ${request.uuid} ]]</td>
<td th:if="${request.report}">
<th:block sec:authorize="not authenticated">[[ ${request.uuid} ]]</th:block>
<a th:href="@{/report/{id}(id=${request.uuid})}" sec:authorize="authenticated">[[ ${request.uuid} ]]</a>
<a th:href="@{/report/{id}(id=${request.uuid})}" sec:authorize="hasRole('USER') or hasRole('ADMIN')">[[ ${request.uuid} ]]</a>
<th:block sec:authorize="not (hasRole('USER') or hasRole('ADMIN'))">[[ ${request.uuid} ]]</th:block>
</td>
<td><time th:datetime="${request.processedAt}">[[ ${request.processedAt} ]]</time></td>
<td class="patient-id" th:if="${patientId != null}" sec:authorize="authenticated">
<td class="patient-id" th:if="${patientId != null}" sec:authorize="hasRole('USER') or hasRole('ADMIN')">
[[ ${request.patientId} ]]
</td>
<td class="patient-id" th:if="${patientId == null}" sec:authorize="authenticated">
<td class="patient-id" th:if="${patientId == null}" sec:authorize="hasRole('USER') or hasRole('ADMIN')">
<a th:href="@{/patient/{pid}(pid=${request.patientId})}">[[ ${request.patientId} ]]</a>
</td>
<td class="patient-id" sec:authorize="not authenticated">***</td>
<td class="patient-id" sec:authorize="not (hasRole('USER') or hasRole('ADMIN'))">***</td>
</tr>
</tbody>
</table>

View File

@ -9,13 +9,15 @@
<div th:replace="~{fragments.html :: nav}"></div>
<main>
<div class="login-form">
<img th:src="@{/user.svg}" alt="user-logo" />
<h2 class="centered">Anmelden</h2>
<div class="centered notification error" th:if="${param.error}">Anmeldung nicht erfolgreich</div>
<div class="centered notification notice" th:if="${param.expired}">Sitzung abgelaufen oder von einem Administrator beendet.</div>
<div class="centered notification success" th:if="${param.logout}">Sie haben sich abgemeldet</div>
<form method="post" th:action="@{/login}">
<input type="text" id="username" name="username" class="form-control" placeholder="Username" required="" autofocus="" />
<input type="password" id="password" name="password" class="form-control" placeholder="Password" required="" />
<button type="submit">Anmelden</button>
<button class="btn" type="submit">Anmelden</button>
<hr th:if="${not oidcLogins.isEmpty()}" />
<a th:each="oidcLogin : ${oidcLogins}" class="btn" th:href="@{/oauth2/authorization/{provider}(provider=${oidcLogin.component1()})}">OIDC Login - [[ ${oidcLogin.component2()} ]]</a>
</form>

View File

@ -0,0 +1,79 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.Consent
import de.ukw.ccc.bwhc.dto.MtbFile
import de.ukw.ccc.bwhc.dto.Patient
import dev.dnpm.etl.processor.services.RequestProcessor
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import org.mockito.ArgumentMatchers.anyString
import org.mockito.Mock
import org.mockito.junit.jupiter.MockitoExtension
import org.mockito.kotlin.any
import org.mockito.kotlin.times
import org.mockito.kotlin.verify
@ExtendWith(MockitoExtension::class)
class KafkaInputListenerTest {
private lateinit var requestProcessor: RequestProcessor
private lateinit var objectMapper: ObjectMapper
private lateinit var kafkaInputListener: KafkaInputListener
@BeforeEach
fun setup(
@Mock requestProcessor: RequestProcessor
) {
this.requestProcessor = requestProcessor
this.objectMapper = ObjectMapper()
this.kafkaInputListener = KafkaInputListener(requestProcessor, objectMapper)
}
@Test
fun shouldProcessMtbFileRequest() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.ACTIVE).build())
.build()
kafkaInputListener.onMessage(ConsumerRecord("testtopic", 0, 0, "", this.objectMapper.writeValueAsString(mtbFile)))
verify(requestProcessor, times(1)).processMtbFile(any())
}
@Test
fun shouldProcessDeleteRequest() {
val mtbFile = MtbFile.builder()
.withPatient(Patient.builder().withId("DUMMY_12345678").build())
.withConsent(Consent.builder().withStatus(Consent.Status.REJECTED).build())
.build()
kafkaInputListener.onMessage(ConsumerRecord("testtopic", 0, 0, "", this.objectMapper.writeValueAsString(mtbFile)))
verify(requestProcessor, times(1)).processDeletion(anyString())
}
}

View File

@ -1,7 +1,7 @@
/*
* This file is part of ETL-Processor
*
* Copyright (c) 2023 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
* Copyright (c) 2024 Comprehensive Cancer Center Mainfranken, Datenintegrationszentrum Philipps-Universität Marburg and Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
@ -17,7 +17,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package dev.dnpm.etl.processor.web
package dev.dnpm.etl.processor.input
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*

View File

@ -21,7 +21,7 @@ package dev.dnpm.etl.processor.output
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.config.KafkaTargetProperties
import dev.dnpm.etl.processor.config.KafkaProperties
import dev.dnpm.etl.processor.monitoring.RequestStatus
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.BeforeEach
@ -53,13 +53,13 @@ class KafkaMtbFileSenderTest {
fun setup(
@Mock kafkaTemplate: KafkaTemplate<String, String>
) {
val kafkaTargetProperties = KafkaTargetProperties("testtopic")
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(1)).build()
this.objectMapper = ObjectMapper()
this.kafkaTemplate = kafkaTemplate
this.kafkaMtbFileSender = KafkaMtbFileSender(kafkaTemplate, kafkaTargetProperties, retryTemplate, objectMapper)
this.kafkaMtbFileSender = KafkaMtbFileSender(kafkaTemplate, kafkaProperties, retryTemplate, objectMapper)
}
@ParameterizedTest
@ -125,9 +125,9 @@ class KafkaMtbFileSenderTest {
@ParameterizedTest
@MethodSource("requestWithResponseSource")
fun shouldRetryOnMtbFileKafkaSendError(testData: TestData) {
val kafkaTargetProperties = KafkaTargetProperties("testtopic")
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(3)).build()
this.kafkaMtbFileSender = KafkaMtbFileSender(this.kafkaTemplate, kafkaTargetProperties, retryTemplate, this.objectMapper)
this.kafkaMtbFileSender = KafkaMtbFileSender(this.kafkaTemplate, kafkaProperties, retryTemplate, this.objectMapper)
doAnswer {
if (null != testData.exception) {
@ -151,9 +151,9 @@ class KafkaMtbFileSenderTest {
@ParameterizedTest
@MethodSource("requestWithResponseSource")
fun shouldRetryOnDeleteKafkaSendError(testData: TestData) {
val kafkaTargetProperties = KafkaTargetProperties("testtopic")
val kafkaProperties = KafkaProperties("testtopic")
val retryTemplate = RetryTemplateBuilder().customPolicy(SimpleRetryPolicy(3)).build()
this.kafkaMtbFileSender = KafkaMtbFileSender(this.kafkaTemplate, kafkaTargetProperties, retryTemplate, this.objectMapper)
this.kafkaMtbFileSender = KafkaMtbFileSender(this.kafkaTemplate, kafkaProperties, retryTemplate, this.objectMapper)
doAnswer {
if (null != testData.exception) {

View File

@ -21,6 +21,7 @@ package dev.dnpm.etl.processor.services
import com.fasterxml.jackson.databind.ObjectMapper
import de.ukw.ccc.bwhc.dto.*
import dev.dnpm.etl.processor.config.AppConfigProperties
import dev.dnpm.etl.processor.monitoring.Request
import dev.dnpm.etl.processor.monitoring.RequestStatus
import dev.dnpm.etl.processor.monitoring.RequestType
@ -51,6 +52,7 @@ class RequestProcessorTest {
private lateinit var sender: MtbFileSender
private lateinit var requestService: RequestService
private lateinit var applicationEventPublisher: ApplicationEventPublisher
private lateinit var appConfigProperties: AppConfigProperties
private lateinit var requestProcessor: RequestProcessor
@ -67,6 +69,7 @@ class RequestProcessorTest {
this.sender = sender
this.requestService = requestService
this.applicationEventPublisher = applicationEventPublisher
this.appConfigProperties = AppConfigProperties(null)
val objectMapper = ObjectMapper()
@ -76,7 +79,8 @@ class RequestProcessorTest {
sender,
requestService,
objectMapper,
applicationEventPublisher
applicationEventPublisher,
appConfigProperties
)
}
@ -390,4 +394,52 @@ class RequestProcessorTest {
assertThat(requestCaptor.firstValue.status).isEqualTo(RequestStatus.ERROR)
}
@Test
fun testShouldNotDetectMtbFileDuplicationIfDuplicationNotConfigured() {
this.appConfigProperties.duplicationDetection = false
doAnswer {
it.arguments[0] as String
}.`when`(pseudonymizeService).patientPseudonym(any())
doAnswer {
it.arguments[0]
}.whenever(transformationService).transform(any())
doAnswer {
MtbFileSender.Response(status = RequestStatus.SUCCESS)
}.`when`(sender).send(any<MtbFileSender.MtbFileRequest>())
val mtbFile = MtbFile.builder()
.withPatient(
Patient.builder()
.withId("1")
.withBirthDate("2000-08-08")
.withGender(Patient.Gender.MALE)
.build()
)
.withConsent(
Consent.builder()
.withId("1")
.withStatus(Consent.Status.ACTIVE)
.withPatient("123")
.build()
)
.withEpisode(
Episode.builder()
.withId("1")
.withPatient("1")
.withPeriod(PeriodStart("2023-08-08"))
.build()
)
.build()
this.requestProcessor.processMtbFile(mtbFile)
val eventCaptor = argumentCaptor<ResponseEvent>()
verify(applicationEventPublisher, times(1)).publishEvent(eventCaptor.capture())
assertThat(eventCaptor.firstValue).isNotNull
assertThat(eventCaptor.firstValue.status).isEqualTo(RequestStatus.SUCCESS)
}
}