View Javadoc
1   package de.dlr.shepard.data.timeseries.repositories;
2   
3   import de.dlr.shepard.common.exceptions.InvalidBodyException;
4   import de.dlr.shepard.common.exceptions.InvalidRequestException;
5   import de.dlr.shepard.data.timeseries.model.TimeseriesDataPoint;
6   import de.dlr.shepard.data.timeseries.model.TimeseriesDataPointsQueryParams;
7   import de.dlr.shepard.data.timeseries.model.enums.AggregateFunction;
8   import de.dlr.shepard.data.timeseries.model.enums.DataPointValueType;
9   import de.dlr.shepard.data.timeseries.model.enums.FillOption;
10  import io.micrometer.core.annotation.Timed;
11  import jakarta.enterprise.context.RequestScoped;
12  import jakarta.persistence.EntityManager;
13  import jakarta.persistence.PersistenceContext;
14  import jakarta.persistence.Query;
15  import java.util.List;
16  import java.util.Optional;
17  import java.util.stream.Collectors;
18  import java.util.stream.IntStream;
19  import org.hibernate.exception.DataException;
20  import org.jspecify.annotations.NonNull;
21  
22  @RequestScoped
23  public class TimeseriesDataPointRepository {
24  
25    private static final int INSERT_BATCH_SIZE = 20000;
26  
27    @PersistenceContext
28    EntityManager entityManager;
29  
30    /**
31     * Insert a list of timeseries data points into the database.
32     *
33     * @param entities         list of timeseries data points
34     * @param timeseriesId The ID of the timeseries
35     * @throws InvalidBodyException can be thrown when 'entities' contains the same
36     *                              timestamp more than once (read more in
37     *                              architectural documentation: 'Building Block
38     *                              View' -> 'Timeseries: Multiple Values for One
39     *                              Timestamp')
40     */
41    @Timed(value = "shepard.timeseries-data-point.batch-insert")
42    public void insertManyDataPoints(
43      List<TimeseriesDataPoint> entities,
44      long timeseriesId,
45      DataPointValueType valueType
46    ) {
47      for (int i = 0; i < entities.size(); i += INSERT_BATCH_SIZE) {
48        int currentLimit = Math.min(i + INSERT_BATCH_SIZE, entities.size());
49        Query query = buildInsertQueryObject(entities, i, currentLimit, timeseriesId, valueType);
50  
51        try {
52          query.executeUpdate();
53        } catch (DataException ex) {
54          if (ex.getCause().toString().contains("ON CONFLICT DO UPDATE command cannot affect row a second time")) {
55            throw new InvalidBodyException(
56              "You provided the same timestamp value multiple times. Please make sure that there are only unique timestamps in a timeseries payload request!"
57            );
58          }
59          throw ex;
60        }
61      }
62    }
63  
64    @Timed(value = "shepard.timeseries-data-point.compression")
65    public void compressAllChunks() {
66      var sqlString = "SELECT compress_chunk(c) FROM show_chunks('timeseries_data_points') c;";
67      Query query = entityManager.createNativeQuery(sqlString);
68      query.getResultList();
69    }
70  
71    /**
72     * Retrieve a list of DataPoints for a time-interval with options to grouping/
73     * time slicing, filling and aggregating.
74     * <p />
75     * This function does not check if the container specified by containerId is
76     * accessible.
77     * We add <code>@ActivateRequestContext</code> in order to call this method in a
78     * parallel stream.
79     *
80     * @param timeseriesId timeseriesId identifying a timeseries
81     * @param valueType type of the timeseries values for column lookup
82     * @param queryParams additional query parameters
83     * @return List<TimeseriesDataPoint>
84     */
85    @Timed(value = "shepard.timeseries-data-point.query")
86    public List<TimeseriesDataPoint> queryDataPoints(
87      long timeseriesId,
88      DataPointValueType valueType,
89      TimeseriesDataPointsQueryParams queryParams
90    ) {
91      assertNotIntegral(queryParams.getFunction());
92      assertCorrectValueTypesForAggregation(queryParams.getFunction(), valueType);
93      assertCorrectValueTypesForFillOption(queryParams.getFillOption(), valueType);
94      assertTimeIntervalForFillOption(queryParams.getTimeSliceNanoseconds(), queryParams.getFillOption());
95      assertAggregationSetForFillOrGrouping(
96        queryParams.getFunction(),
97        queryParams.getTimeSliceNanoseconds(),
98        queryParams.getFillOption()
99      );
100 
101     var query = buildSelectQueryObject(timeseriesId, valueType, queryParams);
102 
103     @SuppressWarnings("unchecked")
104     List<TimeseriesDataPoint> dataPoints = query.getResultList();
105     return dataPoints;
106   }
107 
108   @Timed(value = "shepard.timeseries-data-point-aggregate.query")
109   public List<TimeseriesDataPoint> queryAggregationFunction(
110     long timeseriesId,
111     DataPointValueType valueType,
112     TimeseriesDataPointsQueryParams queryParams
113   ) {
114     assertNotIntegral(queryParams.getFunction());
115     assertCorrectValueTypesForAggregation(queryParams.getFunction(), valueType);
116 
117     var query = buildSelectAggregationFunctionQueryObject(timeseriesId, valueType, queryParams);
118 
119     @SuppressWarnings("unchecked")
120     List<TimeseriesDataPoint> dataPoints = query.getResultList();
121     return dataPoints;
122   }
123 
124   private Query buildInsertQueryObject(
125     List<TimeseriesDataPoint> entities,
126     int startInclusive,
127     int endExclusive,
128     long timeseriesId,
129     DataPointValueType valueType
130   ) {
131     String queryString =
132       "INSERT INTO timeseries_data_points (timeseries_id, time, " +
133       getColumnName(valueType) +
134       ") values " +
135       IntStream.range(startInclusive, endExclusive)
136         .mapToObj(index -> "(:timeseriesid" + ",:time" + index + ",:value" + index + ")")
137         .collect(Collectors.joining(",")) +
138       " ON CONFLICT (timeseries_id, time) DO UPDATE SET time = EXCLUDED.time, timeseries_id = EXCLUDED.timeseries_id, " +
139       getColumnName(valueType) +
140       " = " +
141       "EXCLUDED." +
142       getColumnName(valueType) +
143       ";";
144 
145     Query query = entityManager.createNativeQuery(queryString);
146 
147     query.setParameter("timeseriesid", timeseriesId);
148 
149     IntStream.range(startInclusive, endExclusive).forEach(index -> {
150       query.setParameter("time" + index, entities.get(index).getTimestamp());
151       query.setParameter("value" + index, entities.get(index).getValue());
152     });
153 
154     return query;
155   }
156 
157   private Query buildSelectQueryObject(
158     long timeseriesId,
159     DataPointValueType valueType,
160     TimeseriesDataPointsQueryParams queryParams
161   ) {
162     String columnName = getColumnName(valueType);
163 
164     FillOption fillOption = queryParams.getFillOption().orElse(FillOption.NONE);
165     var timeSliceNanoseconds = queryParams.getTimeSliceNanoseconds().orElse(null);
166 
167     String queryString = "";
168     if (queryParams.getFunction().isPresent()) {
169       AggregateFunction function = queryParams.getFunction().get();
170       if (timeSliceNanoseconds == null) {
171         timeSliceNanoseconds = queryParams.getEndTime() - queryParams.getStartTime();
172       }
173 
174       queryString = "SELECT ";
175 
176       queryString += switch (fillOption) {
177         case NONE -> "time_bucket(:timeInNanoseconds, time) as timestamp, ";
178         case NULL, LINEAR, PREVIOUS -> "time_bucket_gapfill(:timeInNanoseconds, time) as timestamp, ";
179       };
180 
181       String aggregationString = getAggregationString(function, columnName, fillOption);
182 
183       queryString += aggregationString;
184     } else {
185       queryString = "SELECT time, %s ".formatted(columnName);
186     }
187 
188     queryString += """
189     FROM timeseries_data_points
190     WHERE timeseries_id = :timeseriesId
191       AND time >= :startTimeNano
192       AND time <= :endTimeNano
193     """;
194 
195     if (queryParams.getFunction().isPresent()) {
196       queryString += " GROUP BY timestamp ORDER BY timestamp";
197     } else {
198       queryString += " ORDER BY time";
199     }
200 
201     Query query = entityManager.createNativeQuery(queryString, TimeseriesDataPoint.class);
202 
203     if (timeSliceNanoseconds != null) {
204       query.setParameter("timeInNanoseconds", timeSliceNanoseconds);
205     }
206     query.setParameter("timeseriesId", timeseriesId);
207     query.setParameter("startTimeNano", queryParams.getStartTime());
208     query.setParameter("endTimeNano", queryParams.getEndTime());
209 
210     return query;
211   }
212 
213   private static @NonNull String getAggregationString(AggregateFunction function, String columnName) {
214     return getAggregationString(function, columnName, null);
215   }
216 
217   private static @NonNull String getAggregationString(
218     AggregateFunction function,
219     String columnName,
220     FillOption fillOption
221   ) {
222     String aggregationString =
223       switch (function) {
224         case MAX, MIN, COUNT, SUM, STDDEV -> "%s(%s)".formatted(function.name(), columnName);
225         case MEAN -> "AVG(%s)".formatted(columnName);
226         case LAST, FIRST -> "%s(%s, time)".formatted(function.name(), columnName);
227         case SPREAD -> "MAX(%s) - MIN(%s)".formatted(columnName, columnName);
228         case MEDIAN -> "percentile_cont(0.5) WITHIN GROUP (ORDER BY %s)".formatted(columnName);
229         case MODE -> "mode() WITHIN GROUP (ORDER BY %s)".formatted(columnName);
230         case INTEGRAL -> "";
231       };
232 
233     // handle filling - by default bucket_gapfill uses NULL filloption
234     if (fillOption == FillOption.LINEAR) {
235       aggregationString = "interpolate(%s) as value ".formatted(aggregationString);
236     } else if (fillOption == FillOption.PREVIOUS) {
237       aggregationString = "locf(%s) as value ".formatted(aggregationString);
238     } else {
239       aggregationString += " as value ";
240     }
241     return aggregationString;
242   }
243 
244   private Query buildSelectAggregationFunctionQueryObject(
245     long timeseriesId,
246     DataPointValueType valueType,
247     TimeseriesDataPointsQueryParams queryParams
248   ) {
249     String columnName = getColumnName(valueType);
250 
251     String queryString = "";
252     if (queryParams.getFunction().isPresent()) {
253       AggregateFunction function = queryParams.getFunction().get();
254 
255       queryString = "SELECT 1 as timestamp, ";
256 
257       String aggregationString = getAggregationString(function, columnName);
258 
259       queryString += aggregationString;
260     } else {
261       queryString = "SELECT time, %s ".formatted(columnName);
262     }
263 
264     queryString += """
265     FROM timeseries_data_points
266     WHERE timeseries_id = :timeseriesId
267       AND time >= :startTimeNano
268       AND time <= :endTimeNano
269     """;
270 
271     Query query = entityManager.createNativeQuery(queryString, TimeseriesDataPoint.class);
272 
273     query.setParameter("timeseriesId", timeseriesId);
274     query.setParameter("startTimeNano", queryParams.getStartTime());
275     query.setParameter("endTimeNano", queryParams.getEndTime());
276 
277     return query;
278   }
279 
280   private String getColumnName(DataPointValueType valueType) {
281     return switch (valueType) {
282       case Double -> "double_value";
283       case Integer -> "int_value";
284       case String -> "string_value";
285       case Boolean -> "boolean_value";
286     };
287   }
288 
289   /**
290    * Throw when trying to access unsupported aggregation function.
291    */
292   private void assertNotIntegral(Optional<AggregateFunction> function) {
293     if (function.isPresent() && function.get() == AggregateFunction.INTEGRAL) {
294       throw new InvalidRequestException("Aggregation function 'integral' is currently not implemented.");
295     }
296   }
297 
298   /**
299    * Throw when trying to use aggregation functions with boolean or string value
300    * types.
301    * COUNT, FIRST and LAST can be allowed for all data types.
302    */
303   private void assertCorrectValueTypesForAggregation(
304     Optional<AggregateFunction> function,
305     DataPointValueType valueType
306   ) {
307     if (
308       (valueType == DataPointValueType.Boolean || valueType == DataPointValueType.String) &&
309       (function.isPresent() &&
310         function.get() != AggregateFunction.COUNT &&
311         function.get() != AggregateFunction.FIRST &&
312         function.get() != AggregateFunction.LAST)
313     ) {
314       throw new InvalidRequestException(
315         "Cannot execute aggregation functions on data points of type boolean or string."
316       );
317     }
318   }
319 
320   /**
321    * Throw when trying to use gap filling with unsupported value types boolean or
322    * string.
323    */
324   private void assertCorrectValueTypesForFillOption(Optional<FillOption> fillOption, DataPointValueType valueType) {
325     if (
326       (valueType == DataPointValueType.Boolean || valueType == DataPointValueType.String) && (fillOption.isPresent())
327     ) {
328       throw new InvalidRequestException("Cannot use gap filling options on data points of type boolean or string.");
329     }
330   }
331 
332   /**
333    * Throw when trying to use fill option without specifying the timeSlice value
334    */
335   private void assertTimeIntervalForFillOption(Optional<Long> timeSliceNanoseconds, Optional<FillOption> fillOption) {
336     if (timeSliceNanoseconds.isEmpty() && fillOption.isPresent()) {
337       throw new InvalidRequestException("Cannot use gap filling option when no grouping interval is specified.");
338     }
339   }
340 
341   /**
342    * Throw when trying to use fill option or grouping when no aggregation function
343    * is set.
344    */
345   private void assertAggregationSetForFillOrGrouping(
346     Optional<AggregateFunction> function,
347     Optional<Long> timeSliceNanoseconds,
348     Optional<FillOption> fillOption
349   ) {
350     if (function.isEmpty() && (fillOption.isPresent() || timeSliceNanoseconds.isPresent())) {
351       throw new InvalidRequestException(
352         "Cannot use gap filling option or grouping of data when no aggregation function is specified."
353       );
354     }
355   }
356 }