Commit 02d796cd authored by Gerrit Erichsen's avatar Gerrit Erichsen

final state of evaluation.py and other helpers, that started existence along the way

parent bf193538
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 2 08:13:27 2019
@author: Gerrit Erichsen
"""
from os import listdir
from os.path import isfile, join, getctime
import pandas as pd
import time
doContinue = True
basePath = 'I:/Verekon/2017/'
targetPath = 'I:/Verekon/2017/'
blacklist = [] #all already done files
while doContinue:
allFiles = listdir(basePath)
print("Starting (again)")
doContinue = False
for fileName in allFiles:
if isfile(join(basePath, fileName)) \
and fileName.endswith('.csv') \
and not (fileName in blacklist): #\
#and 'WZU' in fileName:
if ((time.time() - getctime(join(basePath, fileName))) < 150.):
doContinue = True
print('Waiting on',fileName)
continue
content = pd.read_csv(join(basePath, fileName), header=None)
print("Reading: ", join(basePath, fileName))
if content.shape[1] == 1:
blacklist.append(fileName)
elif content.shape[1] > 8:
print("Changing file:", fileName)
content = content.drop([0,1,2,3,4,5,6,7], axis=1)
content.to_csv(join(basePath, fileName),
header=False,
index=False,
float_format='%.5f')
doContinue = True
blacklist.append(fileName)
elif content.shape[1] == 5:
print("Changing file:", fileName)
content = content.drop([0,1,2,3], axis=1)
content.to_csv(join(basePath, fileName),
header=False,
index=False,
float_format='%.5f')
doContinue = True
blacklist.append(fileName)
elif content.shape[1] == 3:
print("Changing file:", fileName)
content = content.drop([0,1], axis=1)
content.to_csv(join(basePath, fileName),
header=False,
index=False,
float_format='%.5f')
doContinue = True
blacklist.append(fileName)
else:
blacklist.append(fileName)
time.sleep(60.)
print("Done")
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 3 09:38:06 2019
@author: Gerrit Erichsen
"""
import h5py
import os
import math
def readMetaData(dataType, year, path):
""""(str, str, str) --> ([[str,str,str]])"""
ids = []
fileName = path + '/' + dataType + '/'
if (dataType == 'solar') :
fileName += 'ST'
elif (dataType == 'wind'):
fileName += 'FF'
else:
#assume temperature is supposed to be read
fileName += 'TU'
fileName += '_Stundenwerte_Beschreibung_Stationen.txt.'
file = open(fileName, "r")
content = file.readlines()
file.close()
for line in content:
if line[0].isdigit():
entries = line.split(' ')
#remove
while '' in entries:
entries.remove('')
if not entries[0] == '':
#test if year is within date start and date end
if int(entries[1][:4]) < int(year) \
and int(entries[2][:4]) >= int(year):
#append id, latitude, longitude
end = entries[2]
if int(end[:4]) > 2018 and dataType != 'solar':
end = '20181231'
elif int(end[:4]) > 2018 and dataType == 'solar':
end = '20190808'
ids.append([entries[0], entries[6], entries[4], \
entries[5], entries[1], end])
return ids
def readGrid(fileName):
""""(str) -> ([float], [float])"""
addition = ''
if fileName[-1] != '5':
addition += '_50.h5'
with h5py.File(fileName + addition, 'r') as f:
lat = f['latitude'][:,:]
lon = f['longitude'][:,:]
return lat, lon
def getFullFileName(dataType, stationId, start, end, path):
""""(str, str, str, str, str) --> (str)"""
fileName = path + '/' + dataType + '/stundenwerte_'
#add type qualifier
if (dataType == 'solar') :
fileName += 'ST'
elif (dataType == 'wind'):
fileName += 'FF'
else:
#assume temperature is supposed to be read
fileName += 'TU'
#add general stuff (which is different for solar)
if (dataType == 'solar'):
fileName += '_' + stationId + '_row/'
else:
fileName += '_' + stationId + '_' + start + '_'+ end + '_hist/'
fileName += 'produkt_'
#repeat type qualifiers (as they change letter-case now)
if (dataType == "solar") :
fileName += 'st'
elif (dataType == "wind"):
fileName += 'ff'
else:
#assume temperature is supposed to be read
fileName += 'tu'
#add final file stuff
fileName += '_stunde_' + start + '_' + end + '_' + stationId + '.txt'
if os.path.exists(fileName):
return fileName
return ''
def getFullFileNameH5(dataType,year, path, version = 1):
""""(str, str, str, int) --> (str)"""
fileName = path + '/' + year + '/' + year + '_'
if (dataType == 'solar' and version == 1) :
fileName += 'ASWDIFD'
elif (dataType == 'solar' and version == 2) :
fileName += 'ASWDIR'
elif (dataType == 'wind' and version == 1):
fileName += 'WZU'
elif (dataType == 'wind' and version == 2):
fileName += 'WMV'
else:
#assume temperature is supposed to be read
fileName += 'TMP'
# wind needs additional file accessor, so not finished here
if (dataType != 'wind'):
fileName += '.h5'
return fileName
def findHeight(fileName, identifier, targetYear):
""""(str, str, str) -> (float)"""
index = fileName.rfind('/') + 1
path = fileName[:index]
path += 'Metadaten_Geraete_Windgeschwindigkeit_' + identifier + '.txt'
file = open(path, 'r')
content = file.readlines()
file.close()
for line in content:
if line[0].isdigit() :
entries = line.split(';')
start = int(entries[6][:4])
end = int(entries[7][:4])
if start < int(targetYear) and end >= int(targetYear):
return float(entries[5])
return 10.
def findClosestCell(latGrid, lonGrid, targetLat, targetLon):
""""([[float]],[[float]],float,float)->(int,int)"""
iBest = 0
jBest = 0
currentDiff = calcDistance(latGrid[iBest][jBest], lonGrid[iBest][jBest],\
targetLat, targetLon)
i = 0
iMax = len(latGrid)
while i < iMax:
j = 0
jMax = len(latGrid[i])
while j < jMax:
diff = calcDistance(latGrid[i][j], lonGrid[i][j],\
targetLat, targetLon)
if diff < currentDiff:
iBest = i
jBest = j
currentDiff = diff
j += 1
i += 1
return (iBest, jBest)
def calcDistance(latL, lonL, latR, lonR):
""""(float,float,float,float) -> (float)"""
return math.sqrt((latL - latR)**2 + (lonL - lonR)**2)
\ No newline at end of file
This diff is collapsed.
# -*- coding: utf-8 -*-
"""
Created on Fri Oct 11 07:42:59 2019
@author: Gerrit Erichsen
"""
import h5py
import numpy as np
with h5py.File('D:/WetterdatenPamore/2017/2017_WZU_47.h5', 'r') as f:
dataH5 = np.array(f['WZU'])
dataH52 = []
dataH53 = []
dataH54 = []
with h5py.File('D:/WetterdatenPamore/2017/2017_WMV_47.h5', 'r') as f2:
dataH52 = np.array(f2['WMV'])
with h5py.File('I:/Verekon/2017_WZU_47.h5', 'r') as f3:
dataH53 = np.array(f3['WZU'])
with h5py.File('I:/Verekon/2017_WMV_47.h5', 'r') as f4:
dataH54 = np.array(f4['WMV'])
speeds1 = np.sqrt(dataH5**2 + dataH52**2)
speeds2 = np.sqrt(dataH53**2 + dataH54**2)
diff = speeds1 / speeds2
diff *= -1
diff += 1
print(np.mean(diff))
print(np.max(diff))
print(np.min(diff))
\ No newline at end of file
# -*- coding: utf-8 -*-
"""
Spyder Editor
Dies ist eine temporäre Skriptdatei.
"""
import h5py
import numpy as np
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import cartopy.io.shapereader as shpreader
import dwdfilehelper as dwd
#from shapely.geometry import Point, Polygon
#from geopandas.tools import sjoin
class Coordinate:
lat = 0.
lon = 0.
results = []
def __init__(self, lat, lon):
self.lat = lat
self.lon = lon
self.results = []
if __name__ == '__main__':
allTypes = ['WZU' ,'TMP', 'ASWDIFD'] #data type
coordinates = []
yearStart = 2011
yearEnd = 2017
year = yearStart
path = 'D:/WetterdatenPamore/'
coordinates.append(Coordinate(48.65, 9.28)) #South
coordinates.append(Coordinate(54.20, 9.40)) #North
coordinates.append(Coordinate(50.23, 10.04)) #Center
while year <= yearEnd:
counter = 0
for dType in allTypes:
accessor = path + str(year) + '/' + str(year) + '_'+ dType
if dType == 'WZU': accessor += '_48'
with h5py.File(accessor + '.h5', 'r') as f:
dataH5 = np.array(f[dType])
lat = f['latitude']
lon = f['longitude']
dataH52 = []
if dType == 'WZU':
with h5py.File(path + str(year) + '/' + str(year)
+ '_'+ 'WMV_48' + '.h5', 'r') as f2:
dataH52 = np.array(f2['WMV'])
elif dType == 'ASWDIFD':
with h5py.File(path + str(year) + '/' + str(year)
+ '_'+ 'ASWDIR' + '.h5', 'r') as f2:
dataH52 = np.array(f2['ASWDIR'])
for location in coordinates:
latId, lonId = dwd.findClosestCell(lat, lon, \
location.lat, \
location.lon)
if counter >= len(location.results):
location.results.append([])
data = dataH5[:, latId, lonId]
if dType == 'WZU':
data2 = dataH52[:, latId, lonId]
result = np.sqrt(data**2 + data2**2)
location.results[counter].append(np.mean(result))
elif dType == 'ASWDIFD':
data2 = dataH52[:, latId, lonId]
result = data + data2
location.results[counter].append(np.mean(result))
else:
location.results[counter].append(np.mean(data))
counter += 1
year += 1
#don't know what to do here, so simple print. do anything you want
for location in coordinates:
print(location.results)
\ No newline at end of file
# -*- coding: utf-8 -*-
"""
Spyder Editor
Dies ist eine temporäre Skriptdatei.
"""
import h5py
import numpy as np
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import cartopy.io.shapereader as shpreader
#from shapely.geometry import Point, Polygon
#from geopandas.tools import sjoin
dType = 'TMP' #data type
year = '2013'
path = 'D:/WetterdatenPamore/' + year + '/' \
+ year + '_'+ dType + '.h5'
with h5py.File(path, 'r') as f:
data = f[dType]
lat = f['latitude']
lon = f['longitude']
refPos = (lat[0,0], lon[0,0])
posRight = (lat[0,100], lon[0,100])
posDown = (lat[100,0], lon[100,0])
print (refPos, posRight, posDown)
#print(min(data))
#print(max(data))
#print(data[:15])
print(data[12,0,0])
for key in f.keys():
print(key)
for att in f.attrs.keys():
print(att, f.attrs[att])
data_crs = ccrs.PlateCarree()
fig = plt.figure(figsize=(15,15))
ax = fig.add_subplot(1, 1, 1, projection=ccrs.EuroPP())
ax.set_extent([0, 10, 50, 60], crs=ccrs.PlateCarree())
resolution = '10m'
#ax.add_feature(cfeature.LAND)
#ax.add_feature(cfeature.OCEAN)
ax.add_feature(cfeature.COASTLINE.with_scale(resolution))
ax.add_feature(cfeature.BORDERS.with_scale(resolution))
#ax.add_feature(cfeature.LAKES, alpha=0.5)
#ax.add_feature(cfeature.RIVERS)
ax.gridlines()
ax.scatter(refPos[1], refPos[0], transform=data_crs, c='black')
ax.scatter(posRight[1], posRight[0], transform=data_crs, c='red')
ax.scatter(posDown[1], posDown[0], transform=data_crs, c='blue')
plt.show()
\ No newline at end of file
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 8 10:52:51 2019
@author: Gerrit Erichsen
"""
import h5py
import numpy as np
import math
import matplotlib.pyplot as plt
from windrose import WindroseAxes
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import cartopy.io.shapereader as shpreader
import dwdfilehelper as dwd
from station import WeatherStation
from os.path import join
import seaborn as sns
#from shapely.geometry import Point, Polygon
#from geopandas.tools import sjoin
class Coordinate:
name = ""
lat = 0.
lon = 0.
latId = -1
lonId = -1
results1 = []
results2 = []
def __init__(self, name, lat, lon):
self.name = name
self.lat = lat
self.lon = lon
self.latId = -1
self.lonId = -1
self.results = []
self.results2 = []
if __name__ == '__main__':
dType = 'WZU'
height = 120.
coordinates = []
yearStart = 2007
yearEnd = 2017
year = yearStart
path = 'D:/WetterdatenPamore/'
lower, upper = WeatherStation.getHeightIndices(height)
heightL = WeatherStation.getHeightFromIndex(lower)
if (upper != ''): heightU = WeatherStation.getHeightFromIndex(upper)
#coordinates.append(Coordinate("Buesum", 54.1193, 8.8584)) #South
#coordinates.append(Coordinate("Quickborn", 53.7331, 9.8776)) #North
#coordinates.append(Coordinate("Soltau", 52.9604, 9.7930)) #Center
#coordinates.append(Coordinate("Holzdorf", 51.7657, 13.1666))
#coordinates.append(Coordinate("Roth", 49.2164, 11.1036))
coordinates.append(Coordinate("Fuerstenau", 52.5069, 7.6172))
while year <= yearEnd:
accessor = path + str(year) + '/' + str(year) + '_'+ dType
if dType == 'WZU': accessor += '_' + lower
with h5py.File(accessor + '.h5', 'r') as f:
dataH5 = np.array(f[dType])
lat = f['latitude']
lon = f['longitude']
dataH52 = []
dataH53 = []
dataH54 = []
if dType == 'WZU':
with h5py.File(path + str(year) + '/' + str(year)
+ '_'+ 'WMV_' + lower + '.h5', 'r') as f2:
dataH52 = np.array(f2['WMV'])
if (upper != ''):
with h5py.File(path + str(year) + '/' + str(year)
+ '_'+ 'WZU_' + upper + '.h5', 'r') as f3:
dataH53 = np.array(f3['WZU'])
with h5py.File(path + str(year) + '/' + str(year)
+ '_'+ 'WMV_' + upper + '.h5', 'r') as f4:
dataH54 = np.array(f4['WMV'])
elif dType == 'ASWDIFD':
with h5py.File(path + str(year) + '/' + str(year)
+ '_'+ 'ASWDIR' + '.h5', 'r') as f2:
dataH52 = np.array(f2['ASWDIR'])
for location in coordinates:
if location.latId < 0:
latId, lonId = dwd.findClosestCell(lat, lon, \
location.lat, \
location.lon)
location.latId = latId
location.lonId = lonId
else:
latId = location.latId
lonId = location.lonId
data = np.array(dataH5[:, latId, lonId])
if dType == 'WZU':
data2 = dataH52[:, latId, lonId]
result = np.sqrt(data**2 + data2**2)
result2 = np.arctan2(data, data2)
if (len(dataH53) > 0 and len(dataH54) > 0):
data3 = dataH53[:, latId, lonId]
data4 = dataH54[:, latId, lonId]
factor = (height - heightL) / (heightU - heightL)
result += (np.sqrt(data3**2 + data4**2) - result) \
* factor
result2 = np.arctan2(data + data3 * factor,
data2 + data4 * factor)
result2 *= 180. / math.pi
result2 += 180.
location.results.append(result)
location.results2.append(result2)
elif dType == 'ASWDIFD':
data2 = dataH52[:, latId, lonId]
location.results.append(data)
location.results2.append(data2)
else:
location.results.append(data)
year += 1
print("Done, year ", year - 1)
#don't know what to do here, so simple print. do anything you want
for location in coordinates:
file = open(join("D:/ma_reimer/", location.name + ".csv"), "w")
file.write("speed;direction\n")
for speeds, directions in zip(location.results, location.results2):
for speed, direction in zip(speeds, directions):
line = "%.5f;%.5f \n" % (speed, direction)
file.write(line)
file.close()
print("Done writing ", location.name)
for location in coordinates:
allSpeeds = np.array([])
allDir = np.array([])
counter = 1
title1 = location.name
for speeds, directions in zip(location.results, location.results2):
allSpeeds = np.append(allSpeeds, speeds)
allDir = np.append(allDir, directions)
ax = WindroseAxes.from_ax()
ax.bar(allDir, allSpeeds,
normed=True, opening=0.8, edgecolor='white')
ax.set_yticks(np.arange(0, 15, 5))
ax.set_legend()
plt.title(title1)
plt.figure()
sns.distplot(allSpeeds)
plt.title(title1 + " Dist")
print(location.name, ": avg speed = ", np.mean(allSpeeds))
\ No newline at end of file
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment