16 #include "Base/Axis/PointwiseAxis.h"
17 #include "Device/Coord/AxisNames.h"
24 #include "ui_QREDataLoaderProperties.h"
27 #include <QTextStream>
34 AutoEmit(std::function<
void()> fn)
38 ~AutoEmit() { m_fn(); }
41 std::function<void()> m_fn;
44 QVector<QPair<int, int>> expandLineNumberPattern(
const QString& pattern,
bool* ok =
nullptr)
46 QVector<QPair<int, int>> result;
49 for (
const auto& token : pattern.split(
",")) {
50 const auto parts = token.split(
"-");
56 const auto conv0 = parts[0].toInt(&ok2);
58 const auto conv1 = parts.size() > 1 ? parts[1].toInt(&ok2) : conv0;
60 result.push_back({conv0, conv1});
77 : m_propertiesWidget(nullptr)
84 return "CSV file (Reflectometry - Q/R/sigma_R)";
89 return "QREDataLoader";
97 auto* l =
new QHBoxLayout(parent);
98 l->setContentsMargins(0, 0, 0, 0);
157 QDataStream s(&a, QIODevice::WriteOnly);
158 s.setVersion(QDataStream::Qt_5_12);
183 s.setVersion(QDataStream::Qt_5_12);
208 if (s.status() != QDataStream::Ok)
228 ASSERT(
m_item !=
nullptr);
231 const auto invalidateItemData = [
this]() {
236 const bool parsingSettingsChanged =
240 const bool calculationSettingsChanged =
242 const bool calculationIsNecessary = (parsingSettingsChanged || calculationSettingsChanged);
243 const bool creationOfDatafieldIsNecessary = calculationIsNecessary;
245 if (parsingSettingsChanged)
249 if (calculationIsNecessary)
259 invalidateItemData();
263 if (creationOfDatafieldIsNecessary) {
268 invalidateItemData();
306 const auto isFirstNumberChar = [](
const QChar& c) {
307 return c.isNumber() || c ==
'.' || c ==
'+' || c ==
'-';
310 const auto belongsToNumber = [](
const QChar& c) {
311 return c.isNumber() || c ==
'.' || c ==
'e' || c ==
'E' || c ==
'+' || c ==
'-';
316 const int maxLinesToExamine = 100;
317 while (!in.atEnd() && lineNr < maxLinesToExamine) {
319 QString line = in.readLine().trimmed();
322 if (!isFirstNumberChar(line[0]))
327 while (startOfGap < line.size() && belongsToNumber(line[startOfGap]))
330 if (startOfGap == line.size())
333 int endOfGap = startOfGap;
334 while (endOfGap < line.size() && !belongsToNumber(line[endOfGap]))
337 QStringRef gapContent(&line, startOfGap, endOfGap - startOfGap);
338 if (gapContent.isEmpty())
341 if (gapContent.contains(
"\t")) {
346 gapContent = gapContent.trimmed();
347 if (gapContent.isEmpty()) {
352 if (gapContent.contains(
";")) {
357 if (gapContent.contains(
",")) {
377 const auto lineIsHeader = [headerPrefixes](
const QString& line) {
378 for (
const auto& prefix : headerPrefixes) {
379 if (line.startsWith(prefix.trimmed()))
387 const auto lineShouldBeSkipped = [skippedLines](
int lineNr) {
388 for (
const auto& pair : skippedLines) {
389 if (lineNr >= pair.first && lineNr <= pair.second)
399 QString::SplitBehavior splitBehavior =
402 while (!in.atEnd()) {
405 QString line = in.readLine();
408 lineIsHeader(line) || lineShouldBeSkipped(lineNr) || line.trimmed().isEmpty();
420 const QStringList lineEntries =
426 QVector<double> rowEntriesAsDouble;
428 for (
int col = 0; col < lineEntries.count(); col++) {
430 double val = lineEntries[col].toDouble(&ok);
432 val = std::numeric_limits<double>::quiet_NaN();
434 rowEntriesAsDouble << val;
450 const bool errorColumnIsEnabled = c[
DataType::dR].enabled;
452 const double qFactor = c[
DataType::Q].factor * unitFac;
460 QSet<double> foundQValues;
469 const bool qColIsValid = qCol >= 0 && qCol < rawValues.size();
470 const bool rColIsValid = rCol >= 0 && rCol < rawValues.size();
471 const bool eColIsValid = eCol >= 0 && eCol < rawValues.size();
474 qColIsValid ? rawValues[qCol] * qFactor : std::numeric_limits<double>::quiet_NaN();
476 rColIsValid ? rawValues[rCol] * rFactor : std::numeric_limits<double>::quiet_NaN();
478 eColIsValid ? rawValues[eCol] * eFactor : std::numeric_limits<double>::quiet_NaN();
492 if (std::isnan(e) && errorColumnIsEnabled) {
498 if (foundQValues.contains(q)) {
525 QVector<int> lineNumbers;
529 if (skipLine || lineHasError)
532 lineNumbers.push_back(lineNr);
535 std::sort(lineNumbers.begin(), lineNumbers.end(),
536 [&](
int a,
int b) { return m_importResult.qValues[a] < m_importResult.qValues[b]; });
539 std::vector<double> qVec;
540 std::vector<double> rVec;
542 for (
auto lineNr : lineNumbers) {
546 auto* oData =
new Datafield({
new PointwiseAxis(
"qVector", qVec)}, rVec);
559 auto label_map = DataUtils::AxisNames::specAxis;
560 const auto xAxisTitle = QString::fromStdString(label_map[Coords::QSPACE]);
561 const auto*
const yAxisTitle =
"Signal [a.u.]";
636 calculationErrors.clear();
637 validCalculatedLines = 0;
664 QDataStream s(&a, QIODevice::WriteOnly);
665 s.setVersion(QDataStream::Qt_5_12);
672 s << (quint8)columnDefinitions.count();
673 for (
const auto dataType : columnDefinitions.keys()) {
674 s << (quint8)dataType;
675 s << columnDefinitions[dataType].enabled;
676 s << columnDefinitions[dataType].column;
677 s << (quint8)columnDefinitions[dataType].unit;
678 s << columnDefinitions[dataType].factor;
686 columnDefinitions.clear();
689 s.setVersion(QDataStream::Qt_5_12);
701 for (
int i = 0; i < nDefs; i++) {
704 auto& colDef = columnDefinitions[(
DataType)dataType];
715 if (s.status() != QDataStream::Ok)
721 return enabled ==
other.enabled && column ==
other.column && unit ==
other.unit
722 && factor ==
other.factor;
739 case columnDoesNotContainValidNumber:
740 return QString(
"Raw column %1 does not contain a valid number - line is discarded")
741 .arg(std::get<int>(data));
744 return QString(
"The value %1 for Q is duplicate - line is discarded")
745 .arg(std::get<double>(data));
748 return QString(
"The value %1 for R is greater than 1.0 - line is discarded")
749 .arg(std::get<double>(data));
752 return QString(
"The value %1 for R is less than 0 - line is discarded")
753 .arg(std::get<double>(data));
756 return "Unspecified error";
762 stream << (quint8)s.
type;
763 if (std::holds_alternative<int>(s.
data)) {
765 stream << quint32(std::get<int>(s.
data));
768 stream << qreal(std::get<double>(s.
data));
Defines namespace GUI::Util::CoordName.
Defines class DeserializationException.
Defines class AutomaticMultiColumnDataLoader1DProperties.
Defines class QREDataLoaderResultModel.
QDataStream & operator<<(QDataStream &stream, const QREDataLoader::ImportSettings &s)
QDataStream & operator>>(QDataStream &stream, QREDataLoader::ImportSettings &s)
Defines class QREDataLoader.
Defines class RealDataItem.
Defines class SpecularDataItem.
@ other
The unit has no enum value defined in here (e.g. when defined as an explicit string)
Base class for result tables of data loaders. Derive from this class and return an instance in YourDa...
Abstract base class for all data loaders (classes to import real data).
void contentsProcessed()
Emitted whenever contents have been processed.
void importSettingsChanged()
Emitted whenever an import setting changed.
RealDataItem * m_item
The real-data-item which owns this loader. Never delete this!
Custom property to define list of string values with multiple selections. Intended for QVariant.
void setAxesUnits(const ComboProperty &units)
static DeserializationException streamError()
static DeserializationException tooOld()
static DeserializationException tooNew()
Properties widget for the QREDataLoader.
QByteArray fileContent() const override
Returns the original file content. If not available any more (like for legacy project file import),...
QString persistentClassName() const override
A name which can be used for save/load purposes (which will not change ever more)
friend QDataStream & operator<<(QDataStream &stream, const QREDataLoader::ImportSettings &s)
void guessSettings() override
Guess appropriate settings (for example the separator in a CSV file). Is called only once,...
int numLineRelatedErrors() const override
Number of errors related to a specific line. Such an error means that a particular content (line) can...
int numErrors() const override
Number of errors found while processing the content. An error means that either a particular content ...
void initWithDefaultImportSettings() override
Set import settings to defaults.
void setFileContents(const QByteArray &fileContent) override
Sets the file contents to be imported. If the file was a compressed file, then the decompressed conte...
QByteArray serialize() const override
Returns every internal setting so it can be restored completely.
void calculateFromParseResult() const
void deserialize(const QByteArray &data) override
Initialize from serialization data. If any error occurred, then a DeserializationException has to be ...
void applyImportSettings() override
Read all values from the properties UI into the internal variables.
AbstractDataLoader * clone() const override
Create a complete clone, including all internal states.
AbstractDataLoaderResultModel * createResultModel() const override
Create a table model which contains the import information like original file content,...
void datafieldFromParsingResult(RealDataItem *item) const
void populateImportSettingsWidget(QWidget *parent) override
Fills the widget on the import dialog pane. The implementation here in the base class does nothing (m...
QStringList lineUnrelatedErrors() const override
Errors not related to a particular line.
QString name() const override
The name shown in the format selection combo.
QPointer< QREDataLoaderProperties > m_propertiesWidget
struct QREDataLoader::ImportSettings m_importSettings
ImportResult m_importResult
void parseFileContent() const
void processContents() override
Process the file contents. Can be called more than once, e.g. if the import settings have changed....
friend class QREDataLoaderResultModel
friend QDataStream & operator>>(QDataStream &stream, QREDataLoader::ImportSettings &s)
Provides access to experimental data, for display and fitting. Owns an AbstractDataLoader.
void setNativeDatafield(Datafield *data)
takes ownership of data
bool isSpecularData() const
void setNativeDataUnits(const QString &units)
SpecularDataItem * specularDataItem()
void setYaxisTitle(const QString &title) override
void setDatafield(Datafield *data) override
The given pointer becomes owned by this class!!
void setXaxisTitle(const QString &title) override
void setAxesRangeToData() override
set zoom range of x,y axes to axes of input data
QString nameFromCoord(Coords units)
Returns axes units names from their domain counterpart.
bool operator==(const ColumnDefinition &other) const
Contains a line related error (stored in the import result). Used for showing line related errors in ...
std::variant< int, double > data
Additional data; meaning depends on the error type (see implementation of toString() for more informa...
@ columnDoesNotContainValidNumber
ErrorDefinition(Type t=none, int d=0)
QString toString() const
Human readable error text.
QVector< double > rValues
index is 0-based line number
int maxColumnCount
max found columns in raw data
QString error
error unrelated to lines
void addError(int line, ErrorDefinition::Type type, int data=0)
void clearCalculatedValues()
int validCalculatedLines
number of valid resulting data rows
QVector< QVector< double > > rawValues
index is 0-based line number
QMap< int, ErrorDefinition > calculationErrors
calculation error per line; line is 0-based
QVector< double > eValues
index is 0-based line number
QString errorText(int line) const
ImportSettings importSettings
Settings used for the import.
QVector< double > qValues
index is 0-based line number
QVector< QPair< bool, QString > > lines
bool describes whether line is skipped index is 0-based line number
Settings for importing the file.
QString separator
column separator
QByteArray serialize() const
void deserialize(const QByteArray &data)
bool allowValuesGreaterOne
always true for now (see ##94); could be made configurable later on
QMap< DataType, ColumnDefinition > columnDefinitions
QString headerPrefix
prefix denoting header line
bool operator!=(const ImportSettings &other) const
QString linesToSkip
pattern denoting line to skip (i.e. '1,10-12,42')