1 package de.dlr.shepard.data.timeseries.utilities;
2
3 import com.opencsv.bean.CsvToBeanBuilder;
4 import com.opencsv.bean.StatefulBeanToCsv;
5 import com.opencsv.bean.StatefulBeanToCsvBuilder;
6 import com.opencsv.exceptions.CsvException;
7 import de.dlr.shepard.common.exceptions.InvalidBodyException;
8 import de.dlr.shepard.common.exceptions.InvalidRequestException;
9 import de.dlr.shepard.data.timeseries.io.TimeseriesWithDataPoints;
10 import de.dlr.shepard.data.timeseries.model.Timeseries;
11 import de.dlr.shepard.data.timeseries.model.TimeseriesDataPoint;
12 import io.quarkus.logging.Log;
13 import java.io.IOException;
14 import java.io.InputStream;
15 import java.io.InputStreamReader;
16 import java.io.OutputStreamWriter;
17 import java.nio.charset.StandardCharsets;
18 import java.nio.file.Files;
19 import java.nio.file.Path;
20 import java.util.ArrayList;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Locale;
24 import java.util.stream.Collectors;
25 import org.apache.commons.lang3.math.NumberUtils;
26
27 public final class CsvConverter {
28
29 public static InputStream convertToCsv(Timeseries timeseries, List<TimeseriesDataPoint> dataPoints) {
30 return convertToCsv(List.of(new TimeseriesWithDataPoints(timeseries, dataPoints)));
31 }
32
33 public static InputStream convertToCsv(List<TimeseriesWithDataPoints> timeseriesWithDataPointsList) {
34 Path tmpfile = null;
35 try {
36 tmpfile = Files.createTempFile("shepard", ".csv");
37 try (var stream = Files.newOutputStream(tmpfile); var streamWriter = new OutputStreamWriter(stream)) {
38 StatefulBeanToCsv<CsvTimeseriesDataPoint> writer = new StatefulBeanToCsvBuilder<CsvTimeseriesDataPoint>(
39 streamWriter
40 )
41 .withApplyQuotesToAll(false)
42 .build();
43
44 for (var timeseriesWithDataPoints : timeseriesWithDataPointsList) {
45 try {
46 writer.write(
47 convertTimeseriesWithDataToCsv(
48 timeseriesWithDataPoints.getTimeseries(),
49 timeseriesWithDataPoints.getPoints()
50 )
51 );
52 } catch (CsvException e) {
53 Log.error("CsvException while writing stream", e);
54 throw new InvalidRequestException();
55 }
56 }
57 }
58 } catch (IOException e) {
59 Log.error("IOException while creating or writing to the temp file", e);
60 throw new InvalidRequestException();
61 }
62
63 InputStream result = null;
64 if (tmpfile != null) {
65 try {
66 result = Files.newInputStream(tmpfile);
67 } catch (IOException e) {
68 Log.error("IOException while opening the temp file for reading", e);
69 throw new InvalidRequestException();
70 }
71 }
72 return result;
73 }
74
75 public static List<TimeseriesWithDataPoints> convertToTimeseriesWithData(InputStream stream) {
76 try (var reader = new InputStreamReader(stream)) {
77 var timeseriesDataBuilder = new CsvToBeanBuilder<CsvTimeseriesDataPoint>(reader)
78 .withType(CsvTimeseriesDataPoint.class)
79 .withErrorLocale(Locale.forLanguageTag("en"))
80 .withExceptionHandler(e -> {
81 var encoder = StandardCharsets.ISO_8859_1.newEncoder();
82 var message = encoder.canEncode(e.getMessage()) ? e.getMessage() : "Invalid CSV";
83 Log.errorf("CsvException while reading stream: %s", message);
84 throw new InvalidBodyException(message);
85 })
86 .build();
87
88 List<CsvTimeseriesDataPoint> result = timeseriesDataBuilder.parse();
89
90 return convertCsvToTimeseriesWithData(result);
91 } catch (IOException e) {
92 Log.error("IOException while reading the provided InputStream", e);
93 throw new InvalidRequestException();
94 }
95 }
96
97 private static List<CsvTimeseriesDataPoint> convertTimeseriesWithDataToCsv(
98 Timeseries timeseries,
99 List<TimeseriesDataPoint> dataPoints
100 ) {
101 var result = new ArrayList<CsvTimeseriesDataPoint>(dataPoints.size());
102 for (var dataPoint : dataPoints) {
103 var tsc = new CsvTimeseriesDataPoint(
104 dataPoint.getTimestamp(),
105 timeseries.getMeasurement(),
106 timeseries.getDevice(),
107 timeseries.getLocation(),
108 timeseries.getSymbolicName(),
109 timeseries.getField(),
110 dataPoint.getValue()
111 );
112 result.add(tsc);
113 }
114 return result;
115 }
116
117 private static List<TimeseriesWithDataPoints> convertCsvToTimeseriesWithData(
118 List<CsvTimeseriesDataPoint> csvInputList
119 ) {
120 HashMap<Timeseries, List<TimeseriesDataPoint>> result = new HashMap<Timeseries, List<TimeseriesDataPoint>>();
121
122 for (var csvInputLine : csvInputList) {
123 var timeseries = new Timeseries(
124 csvInputLine.getMeasurement(),
125 csvInputLine.getDevice(),
126 csvInputLine.getLocation(),
127 csvInputLine.getSymbolicName(),
128 csvInputLine.getField()
129 );
130 var dataPoint = new TimeseriesDataPoint(csvInputLine.getTimestamp(), parseValue(csvInputLine.getValue()));
131
132 if (result.containsKey(timeseries)) {
133 result.get(timeseries).add(dataPoint);
134 } else {
135 var dataPoints = new ArrayList<TimeseriesDataPoint>();
136 dataPoints.add(dataPoint);
137 result.put(timeseries, dataPoints);
138 }
139 }
140 List<TimeseriesWithDataPoints> timeseriesWithDataPointsList = result
141 .entrySet()
142 .stream()
143 .map(entry -> new TimeseriesWithDataPoints(entry.getKey(), entry.getValue()))
144 .collect(Collectors.toList());
145 return timeseriesWithDataPointsList;
146 }
147
148 private static Object parseValue(Object input) {
149 List<String> boolString = List.of("true", "false");
150 if (input instanceof String sInput) {
151 if (NumberUtils.isCreatable(sInput)) {
152 try {
153 Integer intValue = Integer.parseInt(sInput);
154 return intValue;
155 } catch (NumberFormatException e) {
156 Double doubleValue = Double.parseDouble(sInput);
157 return doubleValue;
158 }
159 } else if (boolString.contains(sInput.toLowerCase())) {
160 return sInput.equalsIgnoreCase("true");
161 }
162 }
163 return input;
164 }
165 }