1 package de.dlr.shepard.data.timeseries.repositories;
2
3 import de.dlr.shepard.common.exceptions.InvalidBodyException;
4 import de.dlr.shepard.common.exceptions.InvalidRequestException;
5 import de.dlr.shepard.data.timeseries.model.TimeseriesDataPoint;
6 import de.dlr.shepard.data.timeseries.model.TimeseriesDataPointsQueryParams;
7 import de.dlr.shepard.data.timeseries.model.enums.AggregateFunction;
8 import de.dlr.shepard.data.timeseries.model.enums.DataPointValueType;
9 import de.dlr.shepard.data.timeseries.model.enums.FillOption;
10 import io.micrometer.core.annotation.Timed;
11 import jakarta.enterprise.context.RequestScoped;
12 import jakarta.persistence.EntityManager;
13 import jakarta.persistence.PersistenceContext;
14 import jakarta.persistence.Query;
15 import java.util.List;
16 import java.util.Optional;
17 import java.util.stream.Collectors;
18 import java.util.stream.IntStream;
19 import org.hibernate.exception.DataException;
20 import org.jspecify.annotations.NonNull;
21
22 @RequestScoped
23 public class TimeseriesDataPointRepository {
24
25 private static final int INSERT_BATCH_SIZE = 20000;
26
27 @PersistenceContext
28 EntityManager entityManager;
29
30
31
32
33
34
35
36
37
38
39
40
41 @Timed(value = "shepard.timeseries-data-point.batch-insert")
42 public void insertManyDataPoints(
43 List<TimeseriesDataPoint> entities,
44 long timeseriesId,
45 DataPointValueType valueType
46 ) {
47 for (int i = 0; i < entities.size(); i += INSERT_BATCH_SIZE) {
48 int currentLimit = Math.min(i + INSERT_BATCH_SIZE, entities.size());
49 Query query = buildInsertQueryObject(entities, i, currentLimit, timeseriesId, valueType);
50
51 try {
52 query.executeUpdate();
53 } catch (DataException ex) {
54 if (ex.getCause().toString().contains("ON CONFLICT DO UPDATE command cannot affect row a second time")) {
55 throw new InvalidBodyException(
56 "You provided the same timestamp value multiple times. Please make sure that there are only unique timestamps in a timeseries payload request!"
57 );
58 }
59 throw ex;
60 }
61 }
62 }
63
64 @Timed(value = "shepard.timeseries-data-point.compression")
65 public void compressAllChunks() {
66 var sqlString = "SELECT compress_chunk(c) FROM show_chunks('timeseries_data_points') c;";
67 Query query = entityManager.createNativeQuery(sqlString);
68 query.getResultList();
69 }
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85 @Timed(value = "shepard.timeseries-data-point.query")
86 public List<TimeseriesDataPoint> queryDataPoints(
87 long timeseriesId,
88 DataPointValueType valueType,
89 TimeseriesDataPointsQueryParams queryParams
90 ) {
91 assertNotIntegral(queryParams.getFunction());
92 assertCorrectValueTypesForAggregation(queryParams.getFunction(), valueType);
93 assertCorrectValueTypesForFillOption(queryParams.getFillOption(), valueType);
94 assertTimeIntervalForFillOption(queryParams.getTimeSliceNanoseconds(), queryParams.getFillOption());
95 assertAggregationSetForFillOrGrouping(
96 queryParams.getFunction(),
97 queryParams.getTimeSliceNanoseconds(),
98 queryParams.getFillOption()
99 );
100
101 var query = buildSelectQueryObject(timeseriesId, valueType, queryParams);
102
103 @SuppressWarnings("unchecked")
104 List<TimeseriesDataPoint> dataPoints = query.getResultList();
105 return dataPoints;
106 }
107
108 @Timed(value = "shepard.timeseries-data-point-aggregate.query")
109 public List<TimeseriesDataPoint> queryAggregationFunction(
110 long timeseriesId,
111 DataPointValueType valueType,
112 TimeseriesDataPointsQueryParams queryParams
113 ) {
114 assertNotIntegral(queryParams.getFunction());
115 assertCorrectValueTypesForAggregation(queryParams.getFunction(), valueType);
116
117 var query = buildSelectAggregationFunctionQueryObject(timeseriesId, valueType, queryParams);
118
119 @SuppressWarnings("unchecked")
120 List<TimeseriesDataPoint> dataPoints = query.getResultList();
121 return dataPoints;
122 }
123
124 private Query buildInsertQueryObject(
125 List<TimeseriesDataPoint> entities,
126 int startInclusive,
127 int endExclusive,
128 long timeseriesId,
129 DataPointValueType valueType
130 ) {
131 String queryString =
132 "INSERT INTO timeseries_data_points (timeseries_id, time, " +
133 getColumnName(valueType) +
134 ") values " +
135 IntStream.range(startInclusive, endExclusive)
136 .mapToObj(index -> "(:timeseriesid" + ",:time" + index + ",:value" + index + ")")
137 .collect(Collectors.joining(",")) +
138 " ON CONFLICT (timeseries_id, time) DO UPDATE SET time = EXCLUDED.time, timeseries_id = EXCLUDED.timeseries_id, " +
139 getColumnName(valueType) +
140 " = " +
141 "EXCLUDED." +
142 getColumnName(valueType) +
143 ";";
144
145 Query query = entityManager.createNativeQuery(queryString);
146
147 query.setParameter("timeseriesid", timeseriesId);
148
149 IntStream.range(startInclusive, endExclusive).forEach(index -> {
150 query.setParameter("time" + index, entities.get(index).getTimestamp());
151 query.setParameter("value" + index, entities.get(index).getValue());
152 });
153
154 return query;
155 }
156
157 private Query buildSelectQueryObject(
158 long timeseriesId,
159 DataPointValueType valueType,
160 TimeseriesDataPointsQueryParams queryParams
161 ) {
162 String columnName = getColumnName(valueType);
163
164 FillOption fillOption = queryParams.getFillOption().orElse(FillOption.NONE);
165 var timeSliceNanoseconds = queryParams.getTimeSliceNanoseconds().orElse(null);
166
167 String queryString = "";
168 if (queryParams.getFunction().isPresent()) {
169 AggregateFunction function = queryParams.getFunction().get();
170 if (timeSliceNanoseconds == null) {
171 timeSliceNanoseconds = queryParams.getEndTime() - queryParams.getStartTime();
172 }
173
174 queryString = "SELECT ";
175
176 queryString += switch (fillOption) {
177 case NONE -> "time_bucket(:timeInNanoseconds, time) as timestamp, ";
178 case NULL, LINEAR, PREVIOUS -> "time_bucket_gapfill(:timeInNanoseconds, time) as timestamp, ";
179 };
180
181 String aggregationString = getAggregationString(function, columnName, fillOption);
182
183 queryString += aggregationString;
184 } else {
185 queryString = "SELECT time, %s ".formatted(columnName);
186 }
187
188 queryString += """
189 FROM timeseries_data_points
190 WHERE timeseries_id = :timeseriesId
191 AND time >= :startTimeNano
192 AND time <= :endTimeNano
193 """;
194
195 if (queryParams.getFunction().isPresent()) {
196 queryString += " GROUP BY timestamp ORDER BY timestamp";
197 } else {
198 queryString += " ORDER BY time";
199 }
200
201 Query query = entityManager.createNativeQuery(queryString, TimeseriesDataPoint.class);
202
203 if (timeSliceNanoseconds != null) {
204 query.setParameter("timeInNanoseconds", timeSliceNanoseconds);
205 }
206 query.setParameter("timeseriesId", timeseriesId);
207 query.setParameter("startTimeNano", queryParams.getStartTime());
208 query.setParameter("endTimeNano", queryParams.getEndTime());
209
210 return query;
211 }
212
213 private static @NonNull String getAggregationString(AggregateFunction function, String columnName) {
214 return getAggregationString(function, columnName, null);
215 }
216
217 private static @NonNull String getAggregationString(
218 AggregateFunction function,
219 String columnName,
220 FillOption fillOption
221 ) {
222 String aggregationString =
223 switch (function) {
224 case MAX, MIN, COUNT, SUM, STDDEV -> "%s(%s)".formatted(function.name(), columnName);
225 case MEAN -> "AVG(%s)".formatted(columnName);
226 case LAST, FIRST -> "%s(%s, time)".formatted(function.name(), columnName);
227 case SPREAD -> "MAX(%s) - MIN(%s)".formatted(columnName, columnName);
228 case MEDIAN -> "percentile_cont(0.5) WITHIN GROUP (ORDER BY %s)".formatted(columnName);
229 case MODE -> "mode() WITHIN GROUP (ORDER BY %s)".formatted(columnName);
230 case INTEGRAL -> "";
231 };
232
233
234 if (fillOption == FillOption.LINEAR) {
235 aggregationString = "interpolate(%s) as value ".formatted(aggregationString);
236 } else if (fillOption == FillOption.PREVIOUS) {
237 aggregationString = "locf(%s) as value ".formatted(aggregationString);
238 } else {
239 aggregationString += " as value ";
240 }
241 return aggregationString;
242 }
243
244 private Query buildSelectAggregationFunctionQueryObject(
245 long timeseriesId,
246 DataPointValueType valueType,
247 TimeseriesDataPointsQueryParams queryParams
248 ) {
249 String columnName = getColumnName(valueType);
250
251 String queryString = "";
252 if (queryParams.getFunction().isPresent()) {
253 AggregateFunction function = queryParams.getFunction().get();
254
255 queryString = "SELECT 1 as timestamp, ";
256
257 String aggregationString = getAggregationString(function, columnName);
258
259 queryString += aggregationString;
260 } else {
261 queryString = "SELECT time, %s ".formatted(columnName);
262 }
263
264 queryString += """
265 FROM timeseries_data_points
266 WHERE timeseries_id = :timeseriesId
267 AND time >= :startTimeNano
268 AND time <= :endTimeNano
269 """;
270
271 Query query = entityManager.createNativeQuery(queryString, TimeseriesDataPoint.class);
272
273 query.setParameter("timeseriesId", timeseriesId);
274 query.setParameter("startTimeNano", queryParams.getStartTime());
275 query.setParameter("endTimeNano", queryParams.getEndTime());
276
277 return query;
278 }
279
280 private String getColumnName(DataPointValueType valueType) {
281 return switch (valueType) {
282 case Double -> "double_value";
283 case Integer -> "int_value";
284 case String -> "string_value";
285 case Boolean -> "boolean_value";
286 };
287 }
288
289
290
291
292 private void assertNotIntegral(Optional<AggregateFunction> function) {
293 if (function.isPresent() && function.get() == AggregateFunction.INTEGRAL) {
294 throw new InvalidRequestException("Aggregation function 'integral' is currently not implemented.");
295 }
296 }
297
298
299
300
301
302
303 private void assertCorrectValueTypesForAggregation(
304 Optional<AggregateFunction> function,
305 DataPointValueType valueType
306 ) {
307 if (
308 (valueType == DataPointValueType.Boolean || valueType == DataPointValueType.String) &&
309 (function.isPresent() &&
310 function.get() != AggregateFunction.COUNT &&
311 function.get() != AggregateFunction.FIRST &&
312 function.get() != AggregateFunction.LAST)
313 ) {
314 throw new InvalidRequestException(
315 "Cannot execute aggregation functions on data points of type boolean or string."
316 );
317 }
318 }
319
320
321
322
323
324 private void assertCorrectValueTypesForFillOption(Optional<FillOption> fillOption, DataPointValueType valueType) {
325 if (
326 (valueType == DataPointValueType.Boolean || valueType == DataPointValueType.String) && (fillOption.isPresent())
327 ) {
328 throw new InvalidRequestException("Cannot use gap filling options on data points of type boolean or string.");
329 }
330 }
331
332
333
334
335 private void assertTimeIntervalForFillOption(Optional<Long> timeSliceNanoseconds, Optional<FillOption> fillOption) {
336 if (timeSliceNanoseconds.isEmpty() && fillOption.isPresent()) {
337 throw new InvalidRequestException("Cannot use gap filling option when no grouping interval is specified.");
338 }
339 }
340
341
342
343
344
345 private void assertAggregationSetForFillOrGrouping(
346 Optional<AggregateFunction> function,
347 Optional<Long> timeSliceNanoseconds,
348 Optional<FillOption> fillOption
349 ) {
350 if (function.isEmpty() && (fillOption.isPresent() || timeSliceNanoseconds.isPresent())) {
351 throw new InvalidRequestException(
352 "Cannot use gap filling option or grouping of data when no aggregation function is specified."
353 );
354 }
355 }
356 }