Logic for importing intensity data from csv files. More...

Inheritance diagram for CsvImportAssistant:

Collaboration diagram for CsvImportAssistant:

Public Member Functions
	CsvImportAssistant (const QString &file, const bool useGUI=false, QWidget *parent=nullptr)

size_t	columnCount ()

ImportDataInfo	getData ()

char	separator ()

void	setCoordinateColumn (int iCol, Axes::Units units, double multiplier=1.0)

void	setFirstRow (int iRow)

void	setIntensityColumn (int iCol, double multiplier=1.0)

void	setLastRow (int iRow)

Static Public Member Functions
static void	showErrorMessage (std::string message)

Private Member Functions
ImportDataInfo	fillData ()

void	getValuesFromColumns (std::vector< double > &intensityVals, std::vector< double > &coordVals)

char	guessSeparator () const

bool	hasEqualLengthLines (csv::DataArray &dataArray)

bool	loadCsvFile ()

void	removeBlankColumns ()

void	removeMultipleWhiteSpaces ()

void	resetAssistant ()

void	resetSelection ()

void	runDataSelector (QWidget *parent)

Private Attributes
int	m_coordinateColNum

double	m_coordinateMultiplier

csv::DataArray	m_csvArray

std::unique_ptr< CSVFile >	m_csvFile

bool	m_dataAvailable

QString	m_fileName

int	m_firstRow

int	m_intensityColNum

double	m_intensityMultiplier

int	m_lastRow

std::set< int >	m_rowsToDiscard

char	m_separator

Axes::Units	m_units

Detailed Description

Logic for importing intensity data from csv files.

Definition at line 52 of file CsvImportAssistant.h.

Constructor & Destructor Documentation

◆ CsvImportAssistant()

CsvImportAssistant::CsvImportAssistant	(	const QString &	file,
		const bool	useGUI = `false`,
		QWidget *	parent = `nullptr`
	)

Definition at line 30 of file CsvImportAssistant.cpp.

     : m_fileName(file)
     , m_csvFile(nullptr)
     , m_csvArray()
     , m_separator('\0')
     , m_intensityColNum(-1)
     , m_intensityMultiplier(1.0)
     , m_coordinateColNum(-1)
     , m_coordinateMultiplier(1.0)
     , m_firstRow(-1)
     , m_lastRow(-1)
     , m_units(Axes::Units::NBINS)
     , m_dataAvailable(false)
 {
     if (!loadCsvFile()) {
         return;
     }
  
     if (useGUI) {
         runDataSelector(parent);
     } else {
         m_intensityColNum = 0;
         m_coordinateColNum = -1;
         m_units = Axes::Units::NBINS;
         m_firstRow = 0;
         m_lastRow = int(m_csvFile->NumberOfRows() - 1);
         m_dataAvailable = true;
     }
 }

References loadCsvFile(), m_coordinateColNum, m_csvFile, m_dataAvailable, m_firstRow, m_intensityColNum, m_lastRow, m_units, and runDataSelector().

Here is the call graph for this function:

Member Function Documentation

◆ columnCount()

size_t CsvImportAssistant::columnCount ( )

inline

Definition at line 62 of file CsvImportAssistant.h.

62 { return m_csvArray[0].size(); }

References m_csvArray.

◆ fillData()

ImportDataInfo CsvImportAssistant::fillData ( )

private

Definition at line 165 of file CsvImportAssistant.cpp.

 {
     // In case a 2d import is needed in the future
     // Use ArrayUtils::Create2dData(vector<vector<double>>)
     // ArrayUtils::Create2d
     std::unique_ptr<OutputData<double>> resultOutputData;
     resultOutputData = std::make_unique<OutputData<double>>();
     std::vector<double> intensityValues;
     std::vector<double> coordinateValues;
  
     getValuesFromColumns(intensityValues, coordinateValues);
  
     const auto axisName = axisUnitLabel.at(m_units);
     PointwiseAxis coordAxis(axisName, coordinateValues);
     resultOutputData->addAxis(coordAxis);
     resultOutputData->setRawDataVector(intensityValues);
  
     ImportDataInfo result(std::move(resultOutputData), m_units);
     return result;
 }

References axisUnitLabel, getValuesFromColumns(), and m_units.

Referenced by getData().

Here is the call graph for this function:

◆ getData()

ImportDataInfo CsvImportAssistant::getData ( )

inline

Definition at line 56 of file CsvImportAssistant.h.

56 { return m_dataAvailable ? fillData() : ImportDataInfo(); }

CsvImportAssistant::fillData

ImportDataInfo fillData()

Definition: CsvImportAssistant.cpp:165

References fillData(), and m_dataAvailable.

Here is the call graph for this function:

◆ getValuesFromColumns()

void CsvImportAssistant::getValuesFromColumns	(	std::vector< double > &	intensityVals,
		std::vector< double > &	coordVals
	)

private

Definition at line 186 of file CsvImportAssistant.cpp.

 {
     bool intensityOk = true;
     bool coordinateOk = true;
     auto firstRow = size_t(m_firstRow);
     auto lastRow = size_t(m_lastRow) + 1;
     bool isCoordinateNeeded = m_coordinateColNum > -1;
     double intensityValue = 0.;
     double coordinateValue = 0.;
  
     for (size_t row = firstRow; row < lastRow; row++) {
         if (std::find(m_rowsToDiscard.begin(), m_rowsToDiscard.end(), int(row))
             == m_rowsToDiscard.end()) {
             // Intensity Values:
             QString intensityText =
                 QString::fromStdString(m_csvArray[row][size_t(m_intensityColNum)]);
             intensityValue = intensityText.toDouble(&intensityOk);
  
             // Coordinate Values:
             if (isCoordinateNeeded) {
                 QString coordinateText =
                     QString::fromStdString(m_csvArray[row][size_t(m_coordinateColNum)]);
                 coordinateValue = coordinateText.toDouble(&coordinateOk);
             } else {
                 coordinateValue = row - firstRow;
             }
  
             // Add them if they are both ok:
             if (intensityOk && coordinateOk) {
                 intensityValues.push_back(m_intensityMultiplier * intensityValue);
                 coordinateValues.push_back(m_coordinateMultiplier * coordinateValue);
             }
         }
     }
 }

References m_coordinateColNum, m_coordinateMultiplier, m_csvArray, m_firstRow, m_intensityColNum, m_intensityMultiplier, m_lastRow, and m_rowsToDiscard.

Referenced by fillData().

◆ guessSeparator()

char CsvImportAssistant::guessSeparator ( ) const

private

Definition at line 312 of file CsvImportAssistant.cpp.

 {
     int frequencies[127] = {0};
  
     // The actual characters that may be realistically
     // used as separators are only a handfull...
     // And this list seems already exagerated.
     std::vector<char> preferredSeparators;
     preferredSeparators.push_back(' ');
     preferredSeparators.push_back(',');
     preferredSeparators.push_back(';');
     preferredSeparators.push_back('|');
     preferredSeparators.push_back(':');
     preferredSeparators.push_back('\t');
     // preferredSeparators.push_back('/');
     // preferredSeparators.push_back('\\');
     // preferredSeparators.push_back('_');
     preferredSeparators.push_back('\'');
     preferredSeparators.push_back('\"');
  
     // count number of occurences of each char in the file:
     char c;
     std::ifstream is(m_fileName.toStdString());
     while (is.get(c)) {
         if (size_t(c) < 127)
             frequencies[size_t(c)]++;
     }
     is.close();
  
     // set the guessed separator as the most frequent among the
     // preferred separators. -- Some unavoidable hieararchy is
     // present: characters with lower ascii code are preferred.
     char guessedSep = ' ';
     int freq = 0;
     for (char i = 0; i < 127; i++) {
         if (std::find(preferredSeparators.begin(), preferredSeparators.end(), i)
             != preferredSeparators.end())
             if (frequencies[int(i)] > freq) {
                 freq = frequencies[int(i)];
                 guessedSep = i;
             }
     }
  
     // We don't like tabs, as we cannot write them in the GUI.
     // The rest of the CsvImportAssistant and CsvReader should be already aware of this.
     if (guessedSep == '\t')
         guessedSep = ' ';
  
     return guessedSep;
 }

References m_fileName.

Referenced by loadCsvFile(), and runDataSelector().

◆ hasEqualLengthLines()

bool CsvImportAssistant::hasEqualLengthLines ( csv::DataArray & dataArray )

private

Definition at line 363 of file CsvImportAssistant.cpp.

 {
     auto tf = all_of(begin(dataArray), end(dataArray), [dataArray](const csv::DataRow& x) {
         return x.size() == dataArray.front().size();
     });
     return tf;
 }

Referenced by removeBlankColumns().

◆ loadCsvFile()

bool CsvImportAssistant::loadCsvFile ( )

private

Definition at line 112 of file CsvImportAssistant.cpp.

 {
  
     try {
         if (m_separator == '\0')
             m_separator = guessSeparator();
         m_csvFile = std::make_unique<CSVFile>(m_fileName.toStdString(), m_separator);
     } catch (...) {
         showErrorMessage("There was a problem opening the file \"" + m_fileName.toStdString()
                          + "\"");
         return false;
     }
  
     size_t lastRow = m_csvFile->NumberOfRows();
  
     if (lastRow < 1) {
         CsvImportAssistant::showErrorMessage("The file exist but it seems to be empty");
         return false;
     }
  
     auto csvArray = m_csvFile->asArray();
  
     // Automatically ignore empty lines at the end:
     while (QString::fromStdString(accumulate(csvArray[lastRow - 1].begin(),
                                              csvArray[lastRow - 1].end(), std::string("")))
                .trimmed()
            == "") {
         lastRow--;
         if (lastRow < 1) {
             CsvImportAssistant::showErrorMessage("The file exist but it seems to be empty");
             return false;
         }
     }
  
     // TODO: If separator is white space, ignore consecutive spaces...
     // {
     // }
  
     csv::DataArray tmp(csvArray.begin(), csvArray.begin() + int(lastRow));
     m_csvArray.swap(tmp);
     removeBlankColumns();
     if (m_separator == ' ')
         removeMultipleWhiteSpaces();
  
     return true;
 }

References guessSeparator(), m_csvArray, m_csvFile, m_fileName, m_separator, removeBlankColumns(), removeMultipleWhiteSpaces(), and showErrorMessage().

Referenced by CsvImportAssistant(), resetAssistant(), and runDataSelector().

Here is the call graph for this function:

◆ removeBlankColumns()

void CsvImportAssistant::removeBlankColumns ( )

private

Definition at line 261 of file CsvImportAssistant.cpp.

 {
  
     if (m_csvArray.empty())
         return;
  
     csv::DataArray buffer2d;
     csv::DataRow buffer1d;
     std::vector<int> to_be_removed;
  
     size_t nRows = m_csvArray.size();
     size_t nCols = m_csvArray[0].size();
  
     if (!hasEqualLengthLines(m_csvArray)) {
         throw std::runtime_error("All inner vectors should have the same length already.");
     }
  
     // traverse the array columnwise -- this may be inneficient.
     for (size_t j = 0; j < nCols; j++) {
         buffer1d.clear();
         for (size_t i = 0; i < nRows; i++) {
             buffer1d.push_back(m_csvArray[i][j]);
         }
         if (QString::fromStdString(accumulate(buffer1d.begin(), buffer1d.end(), std::string("")))
                 .trimmed()
             == "")
             continue;
  
         buffer2d.push_back(buffer1d);
     }
  
     if (buffer2d.empty()) {
         m_csvArray.clear();
         return;
     }
  
     // now buffer2d has the original array, without blank columns, transposed.
     nCols = buffer2d.size();
     nRows = buffer2d[0].size();
  
     // Save the modified array --i.e. transpose buffer2d
     m_csvArray.clear();
     for (size_t i = 0; i < nRows; i++) {
         buffer1d.clear();
         for (size_t j = 0; j < nCols; j++) {
             buffer1d.push_back(buffer2d[j][i]);
         }
         m_csvArray.push_back(buffer1d);
     }
 }

References hasEqualLengthLines(), and m_csvArray.

Referenced by loadCsvFile().

Here is the call graph for this function:

◆ removeMultipleWhiteSpaces()

void CsvImportAssistant::removeMultipleWhiteSpaces ( )

private

Definition at line 223 of file CsvImportAssistant.cpp.

 {
     if (m_csvArray.empty())
         return;
  
     csv::DataArray buffer2d;
     csv::DataRow buffer1d;
     size_t nRows = m_csvArray.size();
     size_t nCols = m_csvArray[0].size();
     size_t newNcols = 0;
  
     for (size_t i = 0; i < nRows; i++) {
         buffer1d.clear();
         for (size_t j = 0; j < nCols; j++) {
             QString text = QString::fromStdString(m_csvArray[i][j]).trimmed();
             if (text != "")
                 buffer1d.push_back(text.toStdString());
         }
         newNcols = std::max(buffer1d.size(), newNcols);
         buffer2d.push_back(buffer1d);
     }
  
     if (buffer2d.empty()) {
         m_csvArray.clear();
         return;
     }
  
     for (size_t i = 0; i < nRows; i++)
         while (buffer2d[i].size() < newNcols)
             buffer2d[i].push_back("");
  
     // now buffer2d has the original array, without empty cells
     nRows = buffer2d.size();
     nCols = buffer2d[0].size();
  
     m_csvArray.swap(buffer2d);
 }

References m_csvArray.

Referenced by loadCsvFile().

◆ resetAssistant()

void CsvImportAssistant::resetAssistant ( )

private

Definition at line 159 of file CsvImportAssistant.cpp.

 {
     resetSelection();
     loadCsvFile();
 }

References loadCsvFile(), and resetSelection().

Here is the call graph for this function:

◆ resetSelection()

void CsvImportAssistant::resetSelection ( )

private

Definition at line 379 of file CsvImportAssistant.cpp.

 {
     m_csvArray.clear();
     m_intensityColNum = -1;
     m_coordinateColNum = -1;
     m_firstRow = -1;
     m_lastRow = -1;
     m_units = Axes::Units::NBINS;
     m_dataAvailable = false;
 }

References m_coordinateColNum, m_csvArray, m_dataAvailable, m_firstRow, m_intensityColNum, m_lastRow, and m_units.

Referenced by resetAssistant().

◆ runDataSelector()

void CsvImportAssistant::runDataSelector ( QWidget * parent )

private

Definition at line 60 of file CsvImportAssistant.cpp.

 {
     DataSelector selector(m_csvArray, parent);
     m_separator = guessSeparator();
     selector.setSeparator(guessSeparator());
     connect(&selector, &DataSelector::separatorChanged, this, [this, &selector](char newSep) {
         if (newSep != m_separator) {
             m_separator = newSep;
             loadCsvFile();
             selector.setDataArray(m_csvArray);
             selector.setSeparator(newSep);
         }
     });
  
     int res = selector.exec();
  
     if (res == selector.Accepted) {
         m_intensityColNum = selector.intensityColumn();
         m_intensityMultiplier = selector.intensityMultiplier();
         m_coordinateColNum = selector.coordinateColumn();
         m_coordinateMultiplier = selector.coordinateMultiplier();
         m_units = selector.units();
         m_firstRow = int(selector.firstLine() - 1);
         m_lastRow = int(selector.lastLine() - 1);
         m_rowsToDiscard = selector.rowsToDiscard();
         m_dataAvailable = true;
     } else if (res == selector.Rejected) {
         m_dataAvailable = false;
         return;
     }
 }

References DataSelector::coordinateColumn(), DataSelector::coordinateMultiplier(), DataSelector::firstLine(), guessSeparator(), DataSelector::intensityColumn(), DataSelector::intensityMultiplier(), DataSelector::lastLine(), loadCsvFile(), m_coordinateColNum, m_coordinateMultiplier, m_csvArray, m_dataAvailable, m_firstRow, m_intensityColNum, m_intensityMultiplier, m_lastRow, m_rowsToDiscard, m_separator, m_units, DataSelector::rowsToDiscard(), DataSelector::separatorChanged(), DataSelector::setDataArray(), DataSelector::setSeparator(), and DataSelector::units().

Referenced by CsvImportAssistant().

Here is the call graph for this function:

◆ separator()

char CsvImportAssistant::separator ( )

inline

Definition at line 63 of file CsvImportAssistant.h.

63 { return m_separator; }

References m_separator.

◆ setCoordinateColumn()

void CsvImportAssistant::setCoordinateColumn	(	int	iCol,
		Axes::Units	units,
		double	multiplier = `1.0`
	)

Definition at line 97 of file CsvImportAssistant.cpp.

 {
     m_coordinateColNum = iCol - 1;
     m_units = units;
     m_coordinateMultiplier = multiplier;
 }

References m_coordinateColNum, m_coordinateMultiplier, and m_units.

◆ setFirstRow()

void CsvImportAssistant::setFirstRow ( int iRow )

Definition at line 103 of file CsvImportAssistant.cpp.

 {
     m_firstRow = iRow - 1;
 }

References m_firstRow.

◆ setIntensityColumn()

void CsvImportAssistant::setIntensityColumn	(	int	iCol,
		double	multiplier = `1.0`
	)

Definition at line 92 of file CsvImportAssistant.cpp.

 {
     m_intensityColNum = iCol - 1;
     m_intensityMultiplier = multiplier;
 }

References m_intensityColNum, and m_intensityMultiplier.

◆ setLastRow()

void CsvImportAssistant::setLastRow ( int iRow )

Definition at line 107 of file CsvImportAssistant.cpp.

 {
     m_lastRow = iRow - 1;
 }

References m_lastRow.

◆ showErrorMessage()

void CsvImportAssistant::showErrorMessage ( std::string message )

static

Definition at line 371 of file CsvImportAssistant.cpp.

 {
     QMessageBox msgBox;
     msgBox.setText(QString::fromStdString(message));
     msgBox.setIcon(msgBox.Critical);
     msgBox.exec();
 }

Referenced by loadCsvFile(), and DataSelector::updateData().

Member Data Documentation

◆ m_coordinateColNum

int CsvImportAssistant::m_coordinateColNum

private

Definition at line 83 of file CsvImportAssistant.h.

Referenced by CsvImportAssistant(), getValuesFromColumns(), resetSelection(), runDataSelector(), and setCoordinateColumn().

◆ m_coordinateMultiplier

double CsvImportAssistant::m_coordinateMultiplier

private

Definition at line 84 of file CsvImportAssistant.h.

Referenced by getValuesFromColumns(), runDataSelector(), and setCoordinateColumn().

◆ m_csvArray

csv::DataArray CsvImportAssistant::m_csvArray

private

Definition at line 79 of file CsvImportAssistant.h.

Referenced by columnCount(), getValuesFromColumns(), loadCsvFile(), removeBlankColumns(), removeMultipleWhiteSpaces(), resetSelection(), and runDataSelector().

◆ m_csvFile

std::unique_ptr<CSVFile> CsvImportAssistant::m_csvFile

private

Definition at line 78 of file CsvImportAssistant.h.

Referenced by CsvImportAssistant(), and loadCsvFile().

◆ m_dataAvailable

bool CsvImportAssistant::m_dataAvailable

private

Definition at line 89 of file CsvImportAssistant.h.

Referenced by CsvImportAssistant(), getData(), resetSelection(), and runDataSelector().

◆ m_fileName

QString CsvImportAssistant::m_fileName

private

Definition at line 77 of file CsvImportAssistant.h.

Referenced by guessSeparator(), and loadCsvFile().

◆ m_firstRow

int CsvImportAssistant::m_firstRow

private

Definition at line 85 of file CsvImportAssistant.h.

Referenced by CsvImportAssistant(), getValuesFromColumns(), resetSelection(), runDataSelector(), and setFirstRow().

◆ m_intensityColNum

int CsvImportAssistant::m_intensityColNum

private

Definition at line 81 of file CsvImportAssistant.h.

Referenced by CsvImportAssistant(), getValuesFromColumns(), resetSelection(), runDataSelector(), and setIntensityColumn().

◆ m_intensityMultiplier

double CsvImportAssistant::m_intensityMultiplier

private

Definition at line 82 of file CsvImportAssistant.h.

Referenced by getValuesFromColumns(), runDataSelector(), and setIntensityColumn().

◆ m_lastRow

int CsvImportAssistant::m_lastRow

private

Definition at line 86 of file CsvImportAssistant.h.

Referenced by CsvImportAssistant(), getValuesFromColumns(), resetSelection(), runDataSelector(), and setLastRow().

◆ m_rowsToDiscard

std::set<int> CsvImportAssistant::m_rowsToDiscard

private

Definition at line 87 of file CsvImportAssistant.h.

Referenced by getValuesFromColumns(), and runDataSelector().

◆ m_separator

char CsvImportAssistant::m_separator

private

Definition at line 80 of file CsvImportAssistant.h.

Referenced by loadCsvFile(), runDataSelector(), and separator().

◆ m_units

Axes::Units CsvImportAssistant::m_units

private

Definition at line 88 of file CsvImportAssistant.h.

Referenced by CsvImportAssistant(), fillData(), resetSelection(), runDataSelector(), and setCoordinateColumn().

The documentation for this class was generated from the following files:

/home/www/ba/GUI/coregui/Views/ImportDataWidgets/CsvImportAssistant/CsvImportAssistant.h
/home/www/ba/GUI/coregui/Views/ImportDataWidgets/CsvImportAssistant/CsvImportAssistant.cpp

Public Member Functions

Static Public Member Functions

Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ CsvImportAssistant()

Member Function Documentation

◆ columnCount()

◆ fillData()

◆ getData()

◆ getValuesFromColumns()

◆ guessSeparator()

◆ hasEqualLengthLines()

◆ loadCsvFile()

◆ removeBlankColumns()

◆ removeMultipleWhiteSpaces()

◆ resetAssistant()

◆ resetSelection()

◆ runDataSelector()

◆ separator()

◆ setCoordinateColumn()

◆ setFirstRow()

◆ setIntensityColumn()

◆ setLastRow()

◆ showErrorMessage()

Member Data Documentation

◆ m_coordinateColNum

◆ m_coordinateMultiplier

◆ m_csvArray

◆ m_csvFile

◆ m_dataAvailable

◆ m_fileName

◆ m_firstRow

◆ m_intensityColNum

◆ m_intensityMultiplier

◆ m_lastRow

◆ m_rowsToDiscard

◆ m_separator

◆ m_units