Where to drop csv files?

[peterdesmet]

Where to drop csv files?

[stijnvanhoey]

@peterdesmet to abstract away the logic of the path, while keeping it all rely on the filename an additional class is added:

vptstools/src/vptstools/vpts.py

Lines 148 to 232 in d6ec680

	@dataclass(frozen=True)
	class OdimFilePath:
	"""ODIM file path with translation to different s3 key paths"""
	source: str
	radar_code: str
	data_type: str
	year: str
	month: str
	day: str
	hour: str = "00"
	minute: str = "00"
	@classmethod
	def from_file_name(cls, h5_file_path, source):
	""""""
	return cls(source, *cls.parse_file_name(h5_file_path))
	@classmethod
	def from_inventory(cls, h5_file_path):
	""""""
	return cls(h5_file_path.split("/")[0], *cls.parse_file_name(h5_file_path))
	@staticmethod
	def parse_file_name(name):
	"""Parse an hdf5 file name radar_code, year, month, day, hour, minute.
	Parameters
	----------
	name : str
	File name to be parsed. An eventual parent path and
	extension will be removed
	Returns
	-------
	radar_code, data_type, year, month, day, hour, minute
	Notes
	-----
	File format is according to the following file format::
	ccrrr_vp_yyyymmddhhmmss.h5
	with ``c`` the country code two-letter ids and ``rrr``
	the radar three-letter id, e.g. bejab_vp_20161120235500.h5.
	Path information in front of the h5 name itself are ignored.
	"""
	name_regex = re.compile(
	r'.*([^_]{2})([^_]{3})_([^_]*)_(\d\d\d\d)(\d\d)(\d\d)T?'
	r'(\d\d)(\d\d)(?:Z|00)+.*\.h5')
	match = re.match(name_regex, name)
	if match:
	country, radar, data_type, year, \
	month, day, hour, minute = match.groups()
	radar_code = country + radar
	return radar_code, data_type, year, month, day, hour, minute
	else:
	raise ValueError("File name is not a valid ODIM h5 file.")
	@property
	def country(self):
	""""""
	return self.radar_code[:2]
	@property
	def radar(self):
	""""""
	return self.radar_code[2:]
	def _s3_path_setup(self, file_output):
	"""Common setup of the s3 bucket logic"""
	return f"{self.source}/{file_output}/{self.radar_code}/{self.year}"
	@property
	def s3_folder_path_h5(self):
	return f"{self._s3_path_setup('hdf5')}/{self.month}/{self.day}"
	@property
	def s3_file_path_daily_vpts(self):
	return f"{self._s3_path_setup('daily')}/{self.radar_code}_vpts_{self.year}{self.month}{self.day}.csv"
	@property
	def s3_file_path_monthly_vpts(self):
	return f"{self._s3_path_setup('monthly')}/{self.radar_code}_vpts_{self.year}{self.month}.csv"

It contains the logic for s3_folder_path_h5, s3_file_path_daily_vpts and s3_file_path_monthly_vpts as properties and the components are generated from the file name using the parse_file_name method (which can be used outside the context of the class as well to parse in other context as this is a staticmethod).

[stijnvanhoey]

Implemented in #19, closing