pcvolkmer/etl-processor
1
0
Fork 0
mirror of https://github.com/pcvolkmer/etl-processor.git synced 2025-04-19 17:26:51 +00:00
Code Issues Releases Activity

(Near) realtime update of statistics charts

Browse Source
This commit is contained in:
Paul-Christian Volkmer 2023-07-25 20:55:32 +02:00
parent 94846deb98
commit 1a2d4ea7a2
7 changed files with 208 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle.kts
View File

@ -37,6 +37,7 @@ dependencies {
implementation("org.springframework.kafka:spring-kafka")
implementation("org.flywaydb:flyway-mysql")
implementation("commons-codec:commons-codec")
implementation("io.projectreactor.kotlin:reactor-kotlin-extensions")
implementation("de.ukw.ccc:bwhc-dto-java:0.2.0")
runtimeOnly("org.mariadb.jdbc:mariadb-java-client")
runtimeOnly("org.postgresql:postgresql")
@ -45,6 +46,7 @@ dependencies {
annotationProcessor("org.springframework.boot:spring-boot-configuration-processor")
providedRuntime("org.springframework.boot:spring-boot-starter-tomcat")
testImplementation("org.springframework.boot:spring-boot-starter-test")
testImplementation("io.projectreactor:reactor-test")
}
tasks.withType<KotlinCompile> {

9
src/main/kotlin/dev/dnpm/etl/processor/config/AppConfiguration.kt
View File

@ -27,12 +27,16 @@ import dev.dnpm.etl.processor.pseudonym.AnonymizingGenerator
import dev.dnpm.etl.processor.pseudonym.Generator
import dev.dnpm.etl.processor.pseudonym.GpasPseudonymGenerator
import dev.dnpm.etl.processor.pseudonym.PseudonymizeService
import org.reactivestreams.Publisher
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.kafka.core.KafkaTemplate
import reactor.core.publisher.Flux
import reactor.core.publisher.Sinks
import java.net.URI
import java.time.Duration
@Configuration
@EnableConfigurationProperties(
@ -78,5 +82,10 @@ class AppConfiguration {
return KafkaMtbFileSender(kafkaTemplate, objectMapper)
}
@Bean
fun statisticsUpdateProducer(): Sinks.Many<Any> {
return Sinks.many().multicast().directBestEffort()
}
}

7
src/main/kotlin/dev/dnpm/etl/processor/web/MtbFileController.kt
View File

@ -34,13 +34,15 @@ import org.springframework.http.ResponseEntity
import org.springframework.web.bind.annotation.PostMapping
import org.springframework.web.bind.annotation.RequestBody
import org.springframework.web.bind.annotation.RestController
import reactor.core.publisher.Sinks
@RestController
class MtbFileController(
private val pseudonymizeService: PseudonymizeService,
private val senders: List<MtbFileSender>,
private val requestRepository: RequestRepository,
private val objectMapper: ObjectMapper
private val objectMapper: ObjectMapper,
private val statisticsUpdateProducer: Sinks.Many<Any>
) {
private val logger = LoggerFactory.getLogger(MtbFileController::class.java)
@ -63,6 +65,7 @@ class MtbFileController(
report = Report("Duplikat erkannt - keine Daten weitergeleitet")
)
)
statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST)
return ResponseEntity.noContent().build()
}
@ -110,6 +113,8 @@ class MtbFileController(
)
)
statisticsUpdateProducer.emitNext("", Sinks.EmitFailureHandler.FAIL_FAST)
return if (requestStatus == RequestStatus.ERROR) {
ResponseEntity.unprocessableEntity().build()
} else {

5
src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsController.kt
View File

@ -20,15 +20,18 @@
package dev.dnpm.etl.processor.web
import org.springframework.stereotype.Controller
import org.springframework.ui.Model
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.RequestMapping
import java.time.Instant
@Controller
@RequestMapping(path = ["/statistics"])
class StatisticsController {
@GetMapping
fun index(): String {
fun index(model: Model): String {
model.addAttribute("now", Instant.now())
return "statistics"
}

48
src/main/kotlin/dev/dnpm/etl/processor/web/StatisticsRestController.kt
View File

@ -21,9 +21,15 @@ package dev.dnpm.etl.processor.web
import dev.dnpm.etl.processor.monitoring.RequestRepository
import dev.dnpm.etl.processor.monitoring.RequestStatus
import org.reactivestreams.Publisher
import org.springframework.http.MediaType
import org.springframework.http.codec.ServerSentEvent
import org.springframework.web.bind.annotation.GetMapping
import org.springframework.web.bind.annotation.RequestMapping
import org.springframework.web.bind.annotation.RestController
import reactor.core.publisher.Flux
import reactor.core.publisher.Sinks
import reactor.kotlin.core.publisher.toFlux
import java.time.Instant
import java.time.Month
import java.time.ZoneId
@ -34,6 +40,7 @@ import java.time.temporal.TemporalUnit
@RestController
@RequestMapping(path = ["/statistics"])
class StatisticsRestController(
private val statisticsUpdateProducer: Sinks.Many<Any>,
private val requestRepository: RequestRepository
) {
@ -68,13 +75,15 @@ class StatisticsRestController(
.toMap()
Pair(
it.key.toString(),
DateNameValues(it.key.toString(), NameValues(
error = requestList[RequestStatus.ERROR] ?: 0,
warning = requestList[RequestStatus.WARNING] ?: 0,
success = requestList[RequestStatus.SUCCESS] ?: 0,
duplication = requestList[RequestStatus.DUPLICATION] ?: 0,
unknown = requestList[RequestStatus.UNKNOWN] ?: 0,
))
DateNameValues(
it.key.toString(), NameValues(
error = requestList[RequestStatus.ERROR] ?: 0,
warning = requestList[RequestStatus.WARNING] ?: 0,
success = requestList[RequestStatus.SUCCESS] ?: 0,
duplication = requestList[RequestStatus.DUPLICATION] ?: 0,
unknown = requestList[RequestStatus.UNKNOWN] ?: 0,
)
)
)
}.toMap()
@ -86,10 +95,33 @@ class StatisticsRestController(
.sortedBy { it.date }
}
@GetMapping(path = ["events"], produces = [MediaType.TEXT_EVENT_STREAM_VALUE])
fun updater(): Flux<ServerSentEvent<Any>> {
return statisticsUpdateProducer.asFlux().flatMap {
Flux.fromIterable(
listOf(
ServerSentEvent.builder<Any>()
.event("requeststates").id("none").data(this.requestStates())
.build(),
ServerSentEvent.builder<Any>()
.event("requestslastmonth").id("none").data(this.requestsLastMonth())
.build()
)
)
}
}
}
data class NameValue(val name: String, val value: Int, val color: String)
data class DateNameValues(val date: String, val nameValues: NameValues)
data class NameValues(val error: Int = 0, val warning: Int = 0, val success: Int = 0, val duplication: Int = 0, val unknown: Int = 0)
data class NameValues(
val error: Int = 0,
val warning: Int = 0,
val success: Int = 0,
val duplication: Int = 0,
val unknown: Int = 0
)

227
src/main/resources/static/scripts.js
View File

@ -13,101 +13,144 @@ window.onload = () => {
});
};
function drawPieChart(url, elemId, title) {
fetch(url)
.then(resp => resp.json())
.then(data => {
let chartDom = document.getElementById(elemId);
let chart = echarts.init(chartDom);
let option= {
title: {
text: title,
left: 'center'
},
tooltip: {
trigger: 'item'
},
color: data.map(i => i.color),
series: [
{
type: 'pie',
radius: ['40%', '70%'],
avoidLabelOverlap: false,
label: {
show: false,
position: 'center'
},
labelLine: {
show: false
},
data: data
}
]
};
function drawPieChart(url, elemId, title, data) {
if (data) {
update(elemId, data);
} else {
fetch(url)
.then(resp => resp.json())
.then(d => {
draw(elemId, title, d);
update(elemId, d);
});
}
option && chart.setOption(option);
});
function update(elemId, data) {
let chartDom = document.getElementById(elemId);
let chart = echarts.init(chartDom);
let option = {
color: data.map(i => i.color),
animationDuration: 250,
animationDurationUpdate: 250,
series: [
{
type: 'pie',
radius: ['40%', '70%'],
avoidLabelOverlap: false,
label: {
show: false,
position: 'center'
},
labelLine: {
show: false
},
data: data
}
]
};
option && chart.setOption(option);
}
function draw(elemId, title, data) {
let chartDom = document.getElementById(elemId);
let chart = echarts.init(chartDom);
let option= {
title: {
text: title,
left: 'center'
},
tooltip: {
trigger: 'item'
},
color: data.map(i => i.color),
animationDuration: 250,
animationDurationUpdate: 250
};
option && chart.setOption(option);
}
}
function drawBarChart(url, elemId, title) {
fetch(url)
.then(resp => resp.json())
.then(data => {
let chartDom = document.getElementById(elemId);
let chart = echarts.init(chartDom);
let option= {
title: {
text: title,
left: 'center'
},
xAxis: {
type: 'category',
data: data.map(i => dateFormat.format(Date.parse(i.date)))
},
yAxis: {
type: 'value',
minInterval: 2,
},
tooltip: {
trigger: 'item'
},
animation: false,
color: ['slategray', 'red', 'darkorange', 'green', 'slategray'],
series: [
{
name: 'UNKNOWN',
type: 'bar',
stack: 'total',
data: data.map(i => i.nameValues.unknown)
},
{
name: 'ERROR',
type: 'bar',
stack: 'total',
data: data.map(i => i.nameValues.error)
},
{
name: 'WARNING',
type: 'bar',
stack: 'total',
data: data.map(i => i.nameValues.warning)
},
{
name: 'SUCCESS',
type: 'bar',
stack: 'total',
data: data.map(i => i.nameValues.success)
},
{
name: 'DUPLICATION',
type: 'bar',
stack: 'total',
data: data.map(i => i.nameValues.duplication)
}
]
};
function drawBarChart(url, elemId, title, data) {
if (data) {
update(elemId, data);
} else {
fetch(url)
.then(resp => resp.json())
.then(data => {
draw(elemId, title, data);
update(elemId, data);
});
}
option && chart.setOption(option);
});
function update(elemId, data) {
let chartDom = document.getElementById(elemId);
let chart = echarts.init(chartDom);
let option = {
series: [
{
name: 'UNKNOWN',
type: 'bar',
stack: 'total',
data: data.map(i => i.nameValues.unknown)
},
{
name: 'ERROR',
type: 'bar',
stack: 'total',
data: data.map(i => i.nameValues.error)
},
{
name: 'WARNING',
type: 'bar',
stack: 'total',
data: data.map(i => i.nameValues.warning)
},
{
name: 'SUCCESS',
type: 'bar',
stack: 'total',
data: data.map(i => i.nameValues.success)
},
{
name: 'DUPLICATION',
type: 'bar',
stack: 'total',
data: data.map(i => i.nameValues.duplication)
}
]
};
option && chart.setOption(option);
}
function draw(elemId, title, data) {
let chartDom = document.getElementById(elemId);
let chart = echarts.init(chartDom);
let option= {
title: {
text: title,
left: 'center'
},
xAxis: {
type: 'category',
data: data.map(i => dateFormat.format(Date.parse(i.date)))
},
yAxis: {
type: 'value',
minInterval: 1
},
tooltip: {
trigger: 'item'
},
color: ['slategray', 'red', 'darkorange', 'green', 'slategray'],
animationDuration: 250,
animationDurationUpdate: 250
};
option && chart.setOption(option);
}
}

14
src/main/resources/templates/statistics.html
View File

@ -17,8 +17,18 @@
<script th:src="@{/echarts.min.js}"></script>
<script th:src="@{/scripts.js}"></script>
<script>
drawPieChart('statistics/requeststates', 'piechart', 'Statusverteilung der Anfragen');
drawBarChart('statistics/requestslastmonth', 'barchart', 'Anfragen des letzten Monats');
window.onload = () => {
drawPieChart('statistics/requeststates', 'piechart', 'Statusverteilung aller Anfragen');
drawBarChart('statistics/requestslastmonth', 'barchart', 'Anfragen der letzten 30 Tage');
const eventSource = new EventSource('statistics/events');
eventSource.addEventListener('requeststates', event => {
drawPieChart('statistics/requeststates', 'piechart', 'Statusverteilung aller Anfragen', JSON.parse(event.data));
});
eventSource.addEventListener('requestslastmonth', event => {
drawBarChart('statistics/requestslastmonth', 'barchart', 'Anfragen des letzten Monats', JSON.parse(event.data));
});
}
</script>
</body>
</html>