Commit 3b4c198a authored by Béla Wiegel's avatar Béla Wiegel Committed by Gerrit Erichsen
Browse files

Improvement of HDF5 read out performance (CharLGUI)

Removed a bug in HDF5 Reader: multiple distributed Object Header (V1)
Added Wind turbine categorization based on turbine size (PlantDataReaderDialog)
parent be37ca74
......@@ -1194,7 +1194,7 @@ double Wind::calculateDistance(int indexWKA, double windDirection)
double Wind::calculateWakeArea(int indexWKA, double windDirection)
{
int directionAddress = (int)windDirection;
int directionAddress = static_cast<int>(windDirection);
if (directionAddress < 0)
{
directionAddress = 0;
......
......@@ -6,7 +6,9 @@
BinaryFileRead::BinaryFileRead(QString fileLocation) :
m_fileLocation(fileLocation),
m_fileBuffered(false)
m_fileBuffered(false),
m_offset(0),
m_size(0)
{
m_ok = QFile::exists(fileLocation);
}
......@@ -26,6 +28,11 @@ bool BinaryFileRead::dataStream(QDataStream *stream, quint64 offset, quint64 siz
void BinaryFileRead::bufferEntireFile()
{
if (m_fileBuffered)
{
return;
}
std::ifstream file;
file.open(m_fileLocation.toStdString(), std::ios::binary);
if (!file.is_open())
......@@ -47,6 +54,12 @@ void BinaryFileRead::bufferEntireFile()
m_fileBuffered = true;
}
void BinaryFileRead::deleteBufferedData()
{
delete m_bufferedFile;
m_fileBuffered = false;
}
bool BinaryFileRead::fileBuffered() const
{
return m_fileBuffered;
......@@ -63,6 +76,9 @@ bool BinaryFileRead::dataStream(QDataStream * stream, QByteArray * byteArray, qu
size = 100 * 1024 * 1024; // read 100 MB per default
char * byteChunk;
m_offset = offset;
m_size = size;
if (!m_fileBuffered)
{
std::ifstream file;
......
......@@ -12,10 +12,12 @@ public:
bool dataStream(QDataStream * stream, quint64 offset = 0, quint64 size = 0);
bool dataStream(QDataStream * stream, QByteArray * byteArray, quint64 offset = 0, quint64 size = 0);
void bufferEntireFile();
void deleteBufferedData();
bool fileBuffered() const;
bool isOK() const;
quint64 getSize() const {return m_size;}
quint64 getOffset() const {return m_offset;}
private:
bool m_ok;
......@@ -23,6 +25,9 @@ private:
QString m_fileLocation;
bool m_fileBuffered;
char * m_bufferedFile;
quint64 m_offset;
quint64 m_size;
};
#endif // BINARYFILEREAD_H
......@@ -56,7 +56,12 @@ H5Read::H5Read(QString fileLocation) :
* ----------------------------------------------------------------------------------------------- */
// Seek to B-Tree
stream->device()->seek(H_superblock->m_RootGroupSymbolTableEntry->getObjectHeader()->getMessageSymbolTable()->getAddressBTree());
MessageSymbolTable * symbolTable = H_superblock->m_RootGroupSymbolTableEntry->getObjectHeader()->getMessageSymbolTable();
if (symbolTable != nullptr)
stream->device()->seek(symbolTable->getAddressBTree());
else
return;
uchar H_BSignature[4];
const uchar H_BSignature_VALUE[4] = {'T', 'R', 'E', 'E'};
......@@ -348,6 +353,11 @@ void H5Read::bufferEntireFile()
m_binaryFileRead->bufferEntireFile();
}
void H5Read::deleteBufferedData()
{
m_binaryFileRead->deleteBufferedData();
}
pair<float, float> H5Read::getCoordinatesByCellIndex(int index) const
{
return m_LatLonGrid->getPair(index);
......
......@@ -27,6 +27,7 @@ public:
vector<float> extractTimeSeries(QString datasetName, int index) const;
vector<int> getIndicesInPolygon(QPolygonF polygon) const;
void bufferEntireFile();
void deleteBufferedData();
pair<float, float> getCoordinatesByCellIndex(int index) const;
......
......@@ -10,7 +10,8 @@ ObjectHeaderV1::ObjectHeaderV1(QDataStream * stream, BinaryFileRead * binaryFile
m_stream(stream),
m_foundHeaderMessages(0),
m_status(STATUS::NOT_PROCESSED),
m_binaryFileRead(binaryFileRead)
m_binaryFileRead(binaryFileRead),
m_symbolTable(nullptr)
{
m_startBit = stream->device()->pos();
if (HDF5DEBUG) qDebug() << "Begin ObjectHeader:" << m_startBit;
......@@ -31,7 +32,7 @@ ObjectHeaderV1::ObjectHeaderV1(QDataStream * stream, BinaryFileRead * binaryFile
fillUp8ByteBoundary(stream->device()->pos());
}
void ObjectHeaderV1::processStream(bool continued)
void ObjectHeaderV1::processStream(bool continued, quint64 fileOffset)
{
if (!continued)
{
......@@ -49,7 +50,7 @@ void ObjectHeaderV1::processStream(bool continued)
>> sizeOfHeaderMessageData;
quint8 headerMessageFlags;
quint64 temporaryPosition;
quint64 temporaryPosition, temporaryOffset, temporarySize;
switch (messageType)
{
......@@ -95,18 +96,23 @@ void ObjectHeaderV1::processStream(bool continued)
>> continuationLength;
temporaryPosition = m_stream->device()->pos();
temporaryOffset = m_binaryFileRead->getOffset();
temporarySize = m_binaryFileRead->getSize();
// read data from continuationOffset with length continuationLength
m_binaryFileRead->dataStream(m_stream, continuationOffset, continuationLength);
m_stream->device()->seek(0);
this->processStream(true);
if (HDF5DEBUG) qDebug() << "Continue Object Header at file offset:" << continuationOffset;
this->processStream(true, continuationOffset);
if (!m_stream->device()->pos() == continuationLength)
{
qCritical() << _C_ERR << "Didn't reach OffsetLength of Object Header Continuation Block.";
return;
}
m_binaryFileRead->dataStream(m_stream);
if (HDF5DEBUG) qDebug() << "Continuation finished";
m_binaryFileRead->dataStream(m_stream, temporaryOffset, temporarySize);
m_stream->device()->seek(temporaryPosition);
break;
......@@ -176,7 +182,9 @@ void ObjectHeaderV1::processStream(bool continued)
bool ObjectHeaderV1::atEnd()
{
// return m_stream->device()->pos() >= m_startBit + m_size;
return m_foundHeaderMessages >= m_TotalNumberOfHeaderMessages || m_status == FAILURE;
return m_foundHeaderMessages >= m_TotalNumberOfHeaderMessages ||
m_status == FAILURE ||
m_stream->atEnd();
}
quint8 ObjectHeaderV1::fillUp8ByteBoundary(qint64 length)
......
......@@ -16,7 +16,7 @@ class ObjectHeaderV1
public:
ObjectHeaderV1(QDataStream * stream, BinaryFileRead * binaryFileRead);
void processStream(bool continued = false);
void processStream(bool continued = false, quint64 fileOffset = 0);
enum STATUS {NOT_PROCESSED, OK, FAILURE};
STATUS getStatus();
......
......@@ -48,12 +48,16 @@ void CharLGui::onLockConfiguration(bool locked)
qWarning() << "Attention: matching Locations and Nodes failed.";
}
QVector<H5Read*> bufferDataToDelete;
// following constants should be defined in Preferences
const bool getMean = false;
const bool bufferFile = true;
// read data from HDF5 files
// iterate every data type on it's own to reduce memory usage due to file buffering
// ---- WIND ------------------------------------------------------------------------------------------------------
for (int i = 0; i < m_modelLocations->rowCount(); i++)
{
LocationsTableItem * location = m_modelLocations->getMutableItem(i);
......@@ -63,31 +67,60 @@ void CharLGui::onLockConfiguration(bool locked)
// wind data
if (location->getWindDataIndex() >= 0 && location->getWindDataIndex() < m_modelWindData->rowCount())
{
const DataTableItem * dataItem = static_cast<const DataTableItem*>(m_modelWindData->getItem(location->getWindDataIndex()));
if (dataItem->getDataFileH5()->ok() && dataItem->getDataFileTwoH5()->ok())
const DataTableItem * dataItem =
static_cast<const DataTableItem*>(m_modelWindData->getItem(location->getWindDataIndex()));
if (
dataItem->getDataFileH5() != nullptr && dataItem->getDataFileH5()->ok() &&
dataItem->getDataFileTwoH5() != nullptr && dataItem->getDataFileTwoH5()->ok()
)
{
// buffer file
if (bufferFile)
{
dataItem->getDataFileH5()->bufferEntireFile();
dataItem->getDataFileTwoH5()->bufferEntireFile();
if (!bufferDataToDelete.contains(dataItem->getDataFileH5()))
bufferDataToDelete.append(dataItem->getDataFileH5());
if (!bufferDataToDelete.contains(dataItem->getDataFileTwoH5()))
bufferDataToDelete.append(dataItem->getDataFileTwoH5());
}
// zonal wind
vector<float> data = getMean ?
WeatherDataFunctions::readDataMean(*dataItem->getDataFileH5(), WeatherDataFunctions::Wind, 2, location->getMapPolygon()):
WeatherDataFunctions::readDataCentroid(*dataItem->getDataFileH5(), WeatherDataFunctions::Wind, 2, location->getLatitude(), location->getLongitude());
WeatherDataFunctions::readDataMean(
*dataItem->getDataFileH5(), WeatherDataFunctions::Wind, 2,
location->getMapPolygon()) :
WeatherDataFunctions::readDataCentroid(
*dataItem->getDataFileH5(), WeatherDataFunctions::Wind, 2,
static_cast<float>(location->getLatitude()),
static_cast<float>(location->getLongitude()));
getMean ? location->setWeatherWindZMean(data) : location->setWeatherWindZCenter(data);
// meridional wind
data = getMean ?
WeatherDataFunctions::readDataMean(*dataItem->getDataFileTwoH5(), WeatherDataFunctions::Wind, 1, location->getMapPolygon()):
WeatherDataFunctions::readDataCentroid(*dataItem->getDataFileTwoH5(), WeatherDataFunctions::Wind, 1, location->getLatitude(), location->getLongitude());
WeatherDataFunctions::readDataMean(
*dataItem->getDataFileTwoH5(), WeatherDataFunctions::Wind, 1,
location->getMapPolygon()):
WeatherDataFunctions::readDataCentroid(
*dataItem->getDataFileTwoH5(), WeatherDataFunctions::Wind, 1,
static_cast<float>(location->getLatitude()),
static_cast<float>(location->getLongitude()));
getMean ? location->setWeatherWindMMean(data) : location->setWeatherWindMCenter(data);
}
}
}
for (H5Read * h5 : bufferDataToDelete)
h5->deleteBufferedData();
bufferDataToDelete.clear();
// ---- SOLAR -----------------------------------------------------------------------------------------------------
for (int i = 0; i < m_modelLocations->rowCount(); i++)
{
LocationsTableItem * location = m_modelLocations->getMutableItem(i);
......@@ -97,24 +130,60 @@ void CharLGui::onLockConfiguration(bool locked)
// solar data
if (location->getSolarDataIndex() >= 0 && location->getSolarDataIndex() < m_modelSolarData->rowCount())
{
const DataTableItem * dataItem = static_cast<const DataTableItem*>(m_modelSolarData->getItem(location->getSolarDataIndex()));
if (dataItem->getDataFileH5()->ok() && dataItem->getDataFileTwoH5()->ok())
const DataTableItem * dataItem =
static_cast<const DataTableItem*>(m_modelSolarData->getItem(location->getSolarDataIndex()));
if (
dataItem->getDataFileH5() != nullptr && dataItem->getDataFileH5()->ok() &&
dataItem->getDataFileTwoH5() != nullptr && dataItem->getDataFileTwoH5()->ok()
)
{
// buffer file
if (bufferFile)
{
dataItem->getDataFileH5()->bufferEntireFile();
dataItem->getDataFileTwoH5()->bufferEntireFile();
if (!bufferDataToDelete.contains(dataItem->getDataFileH5()))
bufferDataToDelete.append(dataItem->getDataFileH5());
if (!bufferDataToDelete.contains(dataItem->getDataFileTwoH5()))
bufferDataToDelete.append(dataItem->getDataFileTwoH5());
}
// direct irradiation
vector<float> data = getMean ?
WeatherDataFunctions::readDataMean(*dataItem->getDataFileH5(), WeatherDataFunctions::Irradiation, 1, location->getMapPolygon()):
WeatherDataFunctions::readDataCentroid(*dataItem->getDataFileH5(), WeatherDataFunctions::Irradiation, 1, location->getLatitude(), location->getLongitude());
WeatherDataFunctions::readDataMean(
*dataItem->getDataFileH5(), WeatherDataFunctions::Irradiation, 1,
location->getMapPolygon()) :
WeatherDataFunctions::readDataCentroid(
*dataItem->getDataFileH5(), WeatherDataFunctions::Irradiation, 1,
static_cast<float>(location->getLatitude()),
static_cast<float>(location->getLongitude()));
getMean ? location->setWeatherIrrDirMean(data) : location->setWeatherIrrDirCenter(data);
// diffuse irradiation
data = getMean ?
WeatherDataFunctions::readDataMean(*dataItem->getDataFileTwoH5(), WeatherDataFunctions::Irradiation, 2, location->getMapPolygon()):
WeatherDataFunctions::readDataCentroid(*dataItem->getDataFileTwoH5(), WeatherDataFunctions::Irradiation, 2, location->getLatitude(), location->getLongitude());
WeatherDataFunctions::readDataMean(
*dataItem->getDataFileTwoH5(), WeatherDataFunctions::Irradiation, 2,
location->getMapPolygon()):
WeatherDataFunctions::readDataCentroid(
*dataItem->getDataFileTwoH5(), WeatherDataFunctions::Irradiation, 2,
static_cast<float>(location->getLatitude()),
static_cast<float>(location->getLongitude()));
getMean ? location->setWeatherIrrDifMean(data) : location->setWeatherIrrDifCenter(data);
}
}
}
for (H5Read * h5 : bufferDataToDelete)
h5->deleteBufferedData();
bufferDataToDelete.clear();
// ---- TEMPERATURE -----------------------------------------------------------------------------------------------
for (int i = 0; i < m_modelLocations->rowCount(); i++)
{
LocationsTableItem * location = m_modelLocations->getMutableItem(i);
......@@ -124,17 +193,38 @@ void CharLGui::onLockConfiguration(bool locked)
// temperature data
if (location->getTemperatureDataIndex() >= 0 && location->getTemperatureDataIndex() < m_modelTemperatureData->rowCount())
{
const DataTableItem * dataItem = static_cast<const DataTableItem*>(m_modelTemperatureData->getItem(location->getTemperatureDataIndex()));
if (dataItem->getDataFileH5()->ok() && dataItem->getDataFileTwoH5()->ok())
const DataTableItem * dataItem =
static_cast<const DataTableItem*>(m_modelTemperatureData->getItem(location->getTemperatureDataIndex()));
if (dataItem->getDataFileH5() != nullptr && dataItem->getDataFileH5()->ok())
{
// buffer file
if (bufferFile)
{
dataItem->getDataFileH5()->bufferEntireFile();
if (!bufferDataToDelete.contains(dataItem->getDataFileH5()))
bufferDataToDelete.append(dataItem->getDataFileH5());
}
// temperature
vector<float> data = getMean ?
WeatherDataFunctions::readDataMean(*dataItem->getDataFileH5(), WeatherDataFunctions::Temperature, 1, location->getMapPolygon()):
WeatherDataFunctions::readDataCentroid(*dataItem->getDataFileH5(), WeatherDataFunctions::Temperature, 1, location->getLatitude(), location->getLongitude());
WeatherDataFunctions::readDataMean(
*dataItem->getDataFileH5(), WeatherDataFunctions::Temperature, 1,
location->getMapPolygon()):
WeatherDataFunctions::readDataCentroid(
*dataItem->getDataFileH5(), WeatherDataFunctions::Temperature, 1,
static_cast<float>(location->getLatitude()),
static_cast<float>(location->getLongitude()));
getMean ? location->setWeatherTempMean(data) : location->setWeatherTempCenter(data);
}
}
}
for (H5Read * h5 : bufferDataToDelete)
h5->deleteBufferedData();
bufferDataToDelete.clear();
}
void CharLGui::onRunClicked(bool clicked)
......
......@@ -45,6 +45,7 @@ PlantDataReaderDialog::PlantDataReaderDialog(LocationsTableModel * locationsTabl
setLayout(lyoMain);
// temporary
// processOPSDWindData();
// LonLatToAGS();
}
......@@ -190,7 +191,7 @@ void PlantDataReaderDialog::processFile()
nodeItem = static_cast<const NodesTableItem*>(m_nodesTableModel->getItem(nodeID));
found = true;
qDebug() << " found corresponding location" << locationItem->getName() << "and node" << nodeItem->getName() << "by municipality key";
qDebug() << " found corresponding location" << locationItem->getName() << "and node ID" << nodeID << "by municipality key";
break;
}
}
......@@ -314,6 +315,85 @@ void PlantDataReaderDialog::processFile()
}
}
void PlantDataReaderDialog::processOPSDWindData()
{
QList<float> turbinePower = QList<float>({7580, 4500, 5000, 4600, 10000, 500, 1000, 1540, 2000, 3050, 3000, 4200});
qSort(turbinePower.begin(), turbinePower.end());
QList<float> powerClasses;
powerClasses << 0;
for (int i = 0; i < turbinePower.size() - 1; i++)
{
powerClasses << (turbinePower[i] + turbinePower[i + 1]) / 2;
}
QFile file("D:/Locations_Nodes_Plants/OPSD_20190626_Wind.csv");
file.open(QIODevice::ReadOnly);
QString csvHeaderStr = QString::fromUtf8(file.readLine());
csvHeaderStr = csvHeaderStr.simplified();
QList<QString> csvHeader = csvHeaderStr.split(';');
// get csv columns
int indexAGS = csvHeader.indexOf("municipality_code");
int indexPower = csvHeader.indexOf("electrical_capacity");
QMap<QString, QList<float>> powerPerClassinAGS;
while (!file.atEnd())
{
QString csvDataStr = QString::fromUtf8(file.readLine());
csvDataStr = csvDataStr.simplified();
QList<QString> csvData = csvDataStr.split(';');
QString AGS = csvData[indexAGS];
float power = csvData[indexPower].toFloat(); // [MW]
power *= 1000; // [kW]
int powerClass = 0;
while (powerClass < powerClasses.size() - 1 &&
!(power >= powerClasses[powerClass] &&
power < powerClasses[powerClass + 1]))
powerClass++;
if (!powerPerClassinAGS.contains(AGS))
{
powerPerClassinAGS.insert(AGS, QList<float>());
while (powerPerClassinAGS[AGS].size() < powerClasses.size())
powerPerClassinAGS[AGS].append(0);
}
powerPerClassinAGS[AGS][powerClass] += power;
}
file.close();
// write result to csv
QFile result("D:/Locations_Nodes_Plants/WindPowerClasses.csv");
QTextStream stream(&result);
result.open(QIODevice::WriteOnly);
stream << "AGS;";
for (int i = 0; i < turbinePower.size(); i++)
{
stream << QString::number(turbinePower[i]).toInt();
if (i < turbinePower.size() - 1)
stream << ";";
}
stream << endl;
for (QString key : powerPerClassinAGS.keys())
{
stream << key << ";";
for (int i = 0; i < powerPerClassinAGS[key].size(); i++)
{
stream << QString::number(powerPerClassinAGS[key][i]);
if (i < powerPerClassinAGS[key].size() - 1)
stream << ";";
}
stream << endl;
}
result.close();
}
void PlantDataReaderDialog::LonLatToAGS()
{
QFile file("D:/Locations_Nodes_Plants/wind_remaining.csv");
......
......@@ -46,6 +46,7 @@ private:
QString m_filePath;
// temporary
void processOPSDWindData();
void LonLatToAGS();
};
......
......@@ -148,6 +148,12 @@ void LocationsWidget::handleLocationsWidgetCurrentRowChanged(const QItemSelectio
const LocationsTableModel * model = static_cast<const LocationsTableModel *>(current.model());
const LocationsTableItem * item = static_cast<const LocationsTableItem *>(model->getItem(current.row()));
// update data selections
m_solarDataSelection->setCurrentIndex(item->getSolarDataIndex());
m_windDataSelection->setCurrentIndex(item->getWindDataIndex());
m_temperatureDataSelection->setCurrentIndex(item->getTemperatureDataIndex());
m_demandDataSelection->setCurrentIndex(item->getPowerDemandDataIndex());
// get polygons and meta data to display
QPolygonF mapPolygon = item->getMapPolygon();
QVector<QPolygonF> secondaryPolygons = item->getSecondaryPolygons();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment