Coverage for waveqc/views.py: 0%
248 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-05-15 08:47 +0000
1from itertools import groupby
2from typing import Any
4import numpy as np
5import numpy.typing as npt
6import pendulum
7import plotly.graph_objects as go
8from pendulum.date import Date
9from plotly.subplots import make_subplots
10from pyramid.httpexceptions import HTTPNotFound
11from pyramid.request import Request
12from pyramid.response import Response
13from pyramid.view import notfound_view_config, view_config, view_defaults
14from sqlalchemy import Row, extract, func, or_, select
15from sqlalchemy.exc import NoResultFound
16from sqlalchemy.sql.expression import ColumnElement
18from .models import (
19 RESULT_CHANNEL_CLOSED,
20 RESULT_DECONVOLUTION_FAILS,
21 RESULT_DECONVOLUTION_PASS,
22 RESULT_NO_DATA,
23 RESULT_NOT_READABLE,
24 RESULT_PONDERATION,
25 Channel,
26 Check,
27 Network,
28 Operator,
29 Station,
30)
31from .utils import (
32 HEATMAP_LAYOUT_COMPLETENESS,
33 HEATMAP_LAYOUT_NETWORK,
34 HEATMAP_LAYOUT_QUALITY,
35 HEATMAP_LAYOUT_STATION,
36)
38RESULT_COLORS = {
39 RESULT_NO_DATA: "danger",
40 RESULT_NOT_READABLE: "danger",
41 RESULT_DECONVOLUTION_FAILS: "warning",
42 RESULT_DECONVOLUTION_PASS: "success",
43 RESULT_CHANNEL_CLOSED: "dark",
44}
47@notfound_view_config(append_slash=True, renderer="waveqc:templates/404.html")
48def notfound(request: Request) -> dict[Any, Any]:
49 request.response.status = 404
50 return {}
53@view_config(route_name="healthcheck")
54def healthcheck(_request: Request) -> Response:
55 return Response("OK")
58@view_config(route_name="homepage", renderer="templates/homepage.html")
59def homepage(request: Request) -> dict[str, Any]:
60 networks = request.dbsession.scalars(select(Network)).all()
61 stations = request.dbsession.scalars(select(Station).order_by(Station.code)).all()
62 operators = request.dbsession.scalars(
63 select(Operator)
64 .join(Operator.stations)
65 .join(Station.channels)
66 .group_by(Operator.id)
67 .where(Channel.checks.any())
68 .order_by(Operator.agency)
69 ).all()
70 channels = sorted(
71 {
72 channel[:2]
73 for channel in request.dbsession.scalars(
74 select(Channel.code).where(Channel.checks.any()).group_by(Channel.code)
75 ).all()
76 }
77 )
78 return {
79 "networks": networks,
80 "stations": stations,
81 "operators": operators,
82 "channels": channels,
83 }
86@view_config(route_name="network_list", renderer="templates/network_list.html")
87def network_list(request: Request) -> dict[str, Any]:
88 networks = request.dbsession.scalars(select(Network)).all()
89 return {"networks": networks}
92@view_config(route_name="operator_list", renderer="templates/operator_list.html")
93def operator_list(request: Request) -> dict[str, Any]:
94 operators = request.dbsession.scalars(
95 select(Operator)
96 .join(Operator.stations)
97 .join(Station.channels)
98 .group_by(Operator.id)
99 .where(Channel.checks.any())
100 .order_by(Operator.agency)
101 ).all()
102 return {"operators": operators}
105@view_config(route_name="network_summary", renderer="templates/network_summary.html")
106def network_summary(request: Request) -> dict[str, Any]:
107 try:
108 network = request.dbsession.scalars(
109 select(Network).where(Network.code == request.matchdict["code"].upper())
110 ).one()
111 except NoResultFound as exc:
112 raise HTTPNotFound from exc
114 one_week_ago = pendulum.today("utc").subtract(days=7)
115 query = request.dbsession.execute(
116 select(Station, func.min(Check.result))
117 .join(Check.channel)
118 .join(Channel.station)
119 .join(Station.network)
120 .where(
121 Network.code == network.code,
122 Check.date > one_week_ago,
123 extract("year", Station.start_date) <= pendulum.today("utc").year,
124 or_(
125 Channel.end_date == None, # noqa: E711
126 extract("year", Channel.end_date) == pendulum.today("utc").year,
127 ),
128 )
129 .group_by(Station.id)
130 .order_by(Station.code)
131 ).all()
132 stations = [
133 {
134 "code": station[0].code,
135 "id": station[0].id,
136 "result": RESULT_COLORS[station[1]],
137 }
138 for station in query
139 if not station[0].triggered
140 ]
141 triggered_stations = [
142 {
143 "code": station[0].code,
144 "id": station[0].id,
145 "result": RESULT_COLORS[station[1]],
146 }
147 for station in query
148 if station[0].triggered
149 ]
150 return {
151 "network": network,
152 "stations": stations,
153 "triggered_stations": triggered_stations,
154 }
157class HeatmapViews:
158 _full_detail_fields = (
159 Channel.id,
160 Channel.code,
161 Check.date,
162 Check.completeness,
163 Check.result,
164 Check.trace_count,
165 Check.shortest_trace,
166 Channel.location,
167 Station.code,
168 Network.code,
169 )
170 _detail_fields = (
171 Station.code,
172 Station.id,
173 Check.date,
174 func.min(Check.completeness),
175 func.min(Check.result),
176 func.max(Check.trace_count),
177 func.min(Check.shortest_trace),
178 Network.code,
179 )
180 _full_detail_ordering = (
181 Station.code.desc(),
182 Channel.location.desc(),
183 Channel.code.desc(),
184 Check.date,
185 )
186 _detail_ordering = (Station.code.desc(), Check.date)
188 def __init__(self, request: Request) -> None:
189 self.request = request
190 self.view_name = "HeatmapViews"
191 self.context: dict[str, Any] = {}
193 def period(self) -> Any: # noqa: ANN401
194 if "year" in self.request.params:
195 requested_year = int(self.request.params["year"])
196 period = extract("year", Check.date) == requested_year
197 else:
198 requested_year = pendulum.today("utc").year
199 period = Check.date >= pendulum.today("utc").subtract(months=1)
200 return (
201 period,
202 extract("year", Station.start_date) <= requested_year,
203 or_(
204 Channel.end_date == None, # noqa: E711
205 extract("year", Channel.end_date) == requested_year,
206 ),
207 )
209 def triggered(self) -> tuple[ColumnElement[bool]] | tuple[()]:
210 if "triggered" in self.request.params:
211 triggered = bool(int(self.request.params["triggered"]))
212 return (Station.triggered == triggered,)
213 return ()
215 def years_available(self, object_id: int) -> list[int]:
216 start, end = self.request.dbsession.execute(
217 select(func.min(Check.date), func.max(Check.date))
218 .join(Check.channel)
219 .join(Channel.station)
220 .join(Station.network)
221 .where(Network.id == object_id)
222 ).one()
223 return list(range(start.year, end.year + 1))
225 def build_x_values(self, query: list[Row[Any]]) -> list[Date]:
226 # build x values (dates)
227 return sorted({date for date, _ in groupby(query, lambda x: x[2])})
229 def build_y_values_nslc(self, query: list[Row[Any]]) -> list[str]:
230 # build y values (channels)
231 return [
232 nslc for nslc, _ in groupby(query, lambda x: f"{x[9]}.{x[8]}.{x[7]}.{x[1]}")
233 ]
235 def build_y_values_stations(self, query: list[Row[Any]]) -> list[str]:
236 year = self.request.GET.get("year")
237 # include year get parameter if years available
238 parameters = f"?year={ year }" if year else ""
239 # build y values (stations)
240 return [
241 (
242 f"<a href='{ self.request.route_path('station_detail', id=pk) }"
243 f"{parameters}' target='_self'>{network} {station}</a>"
244 )
245 for (pk, station, network), _ in groupby(
246 query, lambda x: (x[1], x[0], x[7])
247 )
248 ]
250 def build_z_values(self, query: list[Row[Any]]) -> npt.NDArray[np.int8]:
251 # build z values (check results)
252 # Warning : Here, we make the assumption that there is a check
253 # for each channel each day
255 # completeness is stored in milliseconds
256 # lets process it in percents
257 time_to_percent_ratio = 100 / (86400 * 1000)
258 return np.array(
259 [
260 [
261 [
262 check[3] * time_to_percent_ratio,
263 RESULT_PONDERATION[check[4]],
264 check[5],
265 check[6],
266 ]
267 for check in checks
268 ]
269 for _, checks in groupby(query, lambda x: x[0])
270 ]
271 )
273 @staticmethod
274 def completeness_trace(mean: npt.NDArray[np.int8]) -> go.Heatmap:
275 return go.Heatmap(
276 y=[
277 (f"<span id='{index}'>{completeness}%</span>")
278 for index, completeness in enumerate(mean)
279 ],
280 z=np.expand_dims(mean, axis=1),
281 **HEATMAP_LAYOUT_COMPLETENESS,
282 )
284 def figure(
285 self, results: npt.NDArray[np.int8], items: list[str], dates: list[Date]
286 ) -> go.Figure:
287 mean = np.round(np.mean(results[:, :, 0], axis=1), 2)
288 # Height = items * 15 px + padding
289 minimum_height = 400
290 layout_height = max(minimum_height, len(items) * 15 + 180)
292 figure = make_subplots(specs=[[{"secondary_y": True}]])
293 figure.add_trace(
294 go.Heatmap(
295 x=dates,
296 y=items,
297 z=results[:, :, 0] + results[:, :, 1],
298 customdata=results,
299 **HEATMAP_LAYOUT_NETWORK,
300 ),
301 secondary_y=False,
302 )
303 figure.add_trace(self.completeness_trace(mean), secondary_y=True)
304 figure.update_layout(
305 xaxis_side="top",
306 height=layout_height,
307 )
308 return figure.to_html(full_html=False, include_plotlyjs=False)
310 def subplots(
311 self, results: npt.NDArray[np.int8], items: list[str], dates: list[Date]
312 ) -> go.Figure:
313 minimum_height = 450
314 layout_height = max(minimum_height, len(items) * 60 + 200)
316 mean = np.round(np.mean(results[:, :, 0], axis=1), 2)
317 figure = make_subplots(
318 rows=2,
319 cols=1,
320 shared_xaxes=True,
321 specs=[[{"secondary_y": True}], [{"secondary_y": False}]],
322 )
323 figure.add_trace(
324 go.Heatmap(
325 x=dates,
326 y=items,
327 z=results[:, :, 0] + results[:, :, 1],
328 customdata=results,
329 **HEATMAP_LAYOUT_QUALITY,
330 ),
331 row=1,
332 col=1,
333 secondary_y=False,
334 )
335 figure.add_trace(self.completeness_trace(mean), secondary_y=True)
336 # We need to know the maximum number of traces for all checks
337 zmax = max(np.max(results[:, :, 2]), 4)
338 HEATMAP_LAYOUT_STATION.update({"zmax": zmax})
339 figure.add_trace(
340 go.Heatmap(x=dates, y=items, z=results[:, :, 2], **HEATMAP_LAYOUT_STATION),
341 row=2,
342 col=1,
343 secondary_y=False,
344 )
345 figure.update_xaxes(side="top")
346 figure.update_layout(height=layout_height)
347 return figure.to_html(full_html=False, include_plotlyjs=False)
349 def context_figure(
350 self,
351 query: list[Row[Any]],
352 items: list[str],
353 *,
354 subplots: bool = False,
355 ) -> None:
356 if not query:
357 # if query is empty, do not try to draw figure
358 figure = None
359 elif subplots:
360 figure = self.subplots(
361 results=self.build_z_values(query),
362 items=items,
363 dates=self.build_x_values(query),
364 )
365 else:
366 figure = self.figure(
367 results=self.build_z_values(query),
368 items=items,
369 dates=self.build_x_values(query),
370 )
371 self.context.update({"figure": figure})
374@view_defaults(renderer="waveqc:templates/station_detail.html")
375class StationViews(HeatmapViews):
376 def years_available(self, object_id: int) -> list[int]:
377 start, end = self.request.dbsession.execute(
378 select(func.min(Check.date), func.max(Check.date))
379 .join(Check.channel)
380 .join(Channel.station)
381 .where(Station.id == object_id)
382 ).one()
383 return list(range(start.year, end.year + 1))
385 def get_station(self) -> Response | tuple[int, str]:
386 try:
387 station_id = int(self.request.matchdict["id"])
388 station_code = self.request.dbsession.scalars(
389 select(Station.code).where(Station.id == station_id)
390 ).one()
391 except (NoResultFound, ValueError) as exc:
392 raise HTTPNotFound from exc
393 return station_id, station_code
395 @view_config(route_name="station_detail")
396 def station_detail(self) -> dict[str, Any]:
397 pk, label = self.get_station()
398 try:
399 query = self.request.dbsession.execute(
400 select(*self._full_detail_fields)
401 .join(Check.channel)
402 .join(Channel.station)
403 .join(Station.network)
404 .where(Station.id == pk, *self.period(), *self.triggered())
405 .order_by(*self._full_detail_ordering)
406 ).all()
407 except NoResultFound as exc:
408 raise HTTPNotFound from exc
410 items = self.build_y_values_nslc(query)
411 self.context_figure(query, items, subplots=True)
412 self.context.update({"label": label})
413 self.context.update({"years_available": self.years_available(pk)})
414 return self.context
417@view_defaults(renderer="waveqc:templates/network_detail.html")
418class NetworkViews(HeatmapViews):
419 def get_network(self) -> Response | tuple[int, str]:
420 network_code = self.request.matchdict["code"].upper()
421 try:
422 network_id = self.request.dbsession.scalars(
423 select(Network.id).where(Network.code == network_code)
424 ).one()
425 except NoResultFound as exc:
426 raise HTTPNotFound from exc
427 return network_id, network_code
429 @view_config(route_name="network_detail")
430 def network_detail(self) -> dict[str, Any]:
431 pk, label = self.get_network()
432 query = self.request.dbsession.execute(
433 select(*self._detail_fields)
434 .join(Check.channel)
435 .join(Channel.station)
436 .join(Station.network)
437 .where(Network.id == pk, *self.period(), *self.triggered())
438 .group_by(Check.date, Network.id, Station.id)
439 .order_by(*self._detail_ordering)
440 ).all()
442 items = self.build_y_values_stations(query)
443 self.context_figure(query, items)
444 self.context.update({"label": label})
445 self.context.update({"years_available": self.years_available(pk)})
446 return self.context
448 @view_config(route_name="network_full_detail")
449 def network_full_detail(self) -> dict[str, Any]:
450 pk, label = self.get_network()
451 query = self.request.dbsession.execute(
452 select(*self._full_detail_fields)
453 .join(Check.channel)
454 .join(Channel.station)
455 .join(Station.network)
456 .where(Network.id == pk, *self.period(), *self.triggered())
457 .order_by(*self._full_detail_ordering)
458 ).all()
460 items = self.build_y_values_nslc(query)
461 self.context_figure(query, items)
462 self.context.update({"label": label})
463 self.context.update({"years_available": self.years_available(pk)})
464 return self.context
467@view_defaults(renderer="waveqc:templates/operator_detail.html")
468class OperatorViews(HeatmapViews):
469 def get_operator(self) -> Response | tuple[int, str]:
470 try:
471 operator_id = int(self.request.matchdict["id"])
472 operator_agency = self.request.dbsession.scalars(
473 select(Operator.agency).where(Operator.id == operator_id)
474 ).one()
475 except (NoResultFound, ValueError) as exc:
476 raise HTTPNotFound from exc
477 return operator_id, operator_agency
479 def years_available(self, object_id: int) -> list[int]:
480 start, end = self.request.dbsession.execute(
481 select(func.min(Check.date), func.max(Check.date))
482 .join(Check.channel)
483 .join(Channel.station)
484 .join(Station.operators)
485 .where(Operator.id == object_id)
486 ).one()
487 return list(range(start.year, end.year + 1))
489 @view_config(route_name="operator_detail")
490 def operator_detail(self) -> dict[str, Any]:
491 pk, label = self.get_operator()
492 query = self.request.dbsession.execute(
493 select(*self._detail_fields)
494 .join(Check.channel)
495 .join(Channel.station)
496 .join(Station.network)
497 .join(Station.operators)
498 .where(Operator.id == pk, *self.period(), *self.triggered())
499 .group_by(Check.date, Network.id, Station.id)
500 .order_by(Network.code.desc(), *self._detail_ordering)
501 ).all()
503 items = self.build_y_values_stations(query)
504 self.context_figure(query, items)
505 self.context.update({"label": label})
506 self.context.update({"years_available": self.years_available(pk)})
507 return self.context
509 @view_config(route_name="operator_full_detail")
510 def operator_full_detail(self) -> dict[str, Any]:
511 pk, label = self.get_operator()
512 query = self.request.dbsession.execute(
513 select(*self._full_detail_fields)
514 .join(Check.channel)
515 .join(Channel.station)
516 .join(Station.network)
517 .join(Station.operators)
518 .where(Operator.id == pk, *self.period(), *self.triggered())
519 .order_by(Network.code.desc(), *self._full_detail_ordering)
520 ).all()
522 items = self.build_y_values_nslc(query)
523 self.context_figure(query, items)
524 self.context.update({"label": label})
525 self.context.update({"years_available": self.years_available(pk)})
526 return self.context
529@view_defaults(renderer="waveqc:templates/channel_detail.html")
530class ChannelViews(HeatmapViews):
531 def __init__(self, request: Request) -> None:
532 super().__init__(request)
533 networks = request.dbsession.scalars(select(Network)).all()
534 stations = request.dbsession.scalars(
535 select(Station).order_by(Station.code)
536 ).all()
537 channels = sorted(
538 {
539 channel[:2]
540 for channel in request.dbsession.scalars(
541 select(Channel.code)
542 .where(Channel.checks.any())
543 .group_by(Channel.code)
544 ).all()
545 }
546 )
547 operators = request.dbsession.scalars(
548 select(Operator)
549 .join(Operator.stations)
550 .join(Station.channels)
551 .group_by(Operator.id)
552 .where(Channel.checks.any())
553 .order_by(Operator.agency)
554 ).all()
555 self.context.update(
556 {
557 "networks": networks,
558 "stations": stations,
559 "channels": channels,
560 "operators": operators,
561 }
562 )
564 def years_available(self, *_: int) -> list[int]:
565 filters = (
566 or_(
567 *(
568 Channel.code.istartswith(value)
569 for value in self.request.GET.getall("channel")
570 )
571 ),
572 )
573 start, end = self.request.dbsession.execute(
574 select(func.min(Check.date), func.max(Check.date))
575 .join(Check.channel)
576 .where(*filters)
577 ).one()
578 return list(range(start.year, end.year + 1))
580 def filters(self) -> Any: # noqa: ANN401
581 filters = self.period()
582 for key in set(self.request.GET.keys()):
583 values = self.request.GET.getall(key)
584 if values:
585 match key:
586 case "network":
587 filters += (Network.code.in_(values),)
588 case "station":
589 filters += (Station.code.in_(values),)
590 case "channel":
591 filters += (
592 or_(*(Channel.code.istartswith(value) for value in values)),
593 )
594 case "operator":
595 filters += (
596 or_(
597 *(Operator.agency.icontains(value) for value in values)
598 ),
599 )
600 return filters
602 def label(self) -> str:
603 return " - ".join(
604 [
605 f"{key.title()}: {value.upper()}"
606 for key, value in self.request.GET.items()
607 ]
608 )
610 @view_config(route_name="channel_detail")
611 def channel_detail(self) -> dict[str, Any]:
612 query = self.request.dbsession.execute(
613 select(*self._detail_fields)
614 .join(Check.channel)
615 .join(Channel.station)
616 .join(Station.network)
617 .join(Station.operators)
618 .where(*self.triggered(), *self.filters())
619 .group_by(Check.date, Network.id, Station.id)
620 .order_by(Network.code.desc(), *self._detail_ordering)
621 ).all()
623 items = self.build_y_values_stations(query)
624 self.context_figure(query, items)
625 self.context.update({"label": "Channels"})
626 self.context.update({"years_available": self.years_available()})
627 return self.context
629 @view_config(route_name="channel_full_detail")
630 def channel_full_detail(self) -> dict[str, Any]:
631 stmt = (
632 select(*self._full_detail_fields)
633 .join(Check.channel)
634 .join(Channel.station)
635 .join(Station.network)
636 )
637 if self.request.GET.get("operator"):
638 stmt = stmt.join(Station.operators)
639 query = self.request.dbsession.execute(
640 stmt.where(*self.triggered(), *self.filters()).order_by(
641 Network.code.desc(), *self._full_detail_ordering
642 )
643 ).all()
644 items = self.build_y_values_nslc(query)
645 self.context_figure(query, items)
646 self.context.update({"label": "Channels"})
647 self.context.update({"years_available": self.years_available()})
648 return self.context