3 A class for determining the OBPG type of a file.
7 __author__ =
'melliott'
23 KNOWN_SENSORS = [
'Aquarius',
'CZCS',
'HAWKEYE',
'HICO',
24 'HMODISA',
'HMODIST',
'MERIS',
'MODISA',
25 'MODIS Aqua',
'MODIST',
'MODIS Terra',
26 'MOS',
'OCIS',
'OCM2',
'OCTS',
'OSMI',
27 'SeaWiFS',
'SGLI',
'VIIRSN',
'VIIRSJ1']
29 MONTH_ABBRS = dict((v.upper(), k)
for k, v
in enumerate(calendar.month_abbr))
34 Converts a number of milliseconds to a string representing the time of day.
36 secs = math.trunc(millisecs / 1000)
37 hrs = math.trunc(secs / 3600)
38 secs = secs - (hrs * 3600)
39 mins = math.trunc(secs / 60)
40 secs = secs - (mins * 60)
41 tm_str =
'{0:02}{1:02}{2:02}'.format(hrs, mins, secs)
46 Creates timestamp for VIIRS L1.
48 year =
int(sdate[0:4])
52 mins =
int(stime[2:4])
53 secs =
int(stime[4:6])
54 dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
55 return dt_obj.strftime(
'%Y%j%H%M%S')
59 Return a timestamp in the YYYYDDDHHMMSS form from a given year (yr), day
60 of year (doy) and milliseconds (after 00:00 - millisecs).
63 timestamp =
'{0:04}{1:03}{2}'.format(year, doy, time_str)
68 A class containing methods for finding the type of an OBPG file (e.g.
69 MODIS L2b, SeaWiFS L1A, Aquarius L3m, etc.
74 Save the path of the file in question and set up default
75 values for several thing still to be found.
77 if os.path.exists(fpath):
78 if tarfile.is_tarfile(fpath):
82 err_msg =
"Error! Cannot process file {0}.".format(fpath)
93 err_msg =
"Error! File {0} could not be found.".format(fpath)
109 def _create_modis_l1_timestamp(self, rng_date, rng_time):
111 Returns a date/time stamp for a MODIS L1 file from the appropriate
112 RANGINGDATE and RANGINGTIME attributes. The returned date/time stamp
113 is of form YYYYDDDHHMMSS, where YYYY = year, DDD = day of year,
114 HH = hour, MM = minute, and SS = second.
116 year =
int(rng_date[0:4])
117 mon =
int(rng_date[5:7])
118 dom =
int(rng_date[8:10])
119 hrs =
int(rng_time[0:2])
120 mins =
int(rng_time[3:5])
121 secs =
int(rng_time[6:8])
122 milsecs =
int(rng_time[9:15])
123 dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs, milsecs)
124 if milsecs >= 500000:
125 dt_delta = datetime.timedelta(seconds=1)
127 return dt_obj.strftime(
'%Y%j%H%M%S')
129 def _create_octs_l1_timestamp(self, dt_str):
131 Creates a timestamp for an OCTS L1.
133 year =
int(dt_str[0:4])
134 mon =
int(dt_str[4:6])
135 dom =
int(dt_str[6:8])
136 hrs =
int(dt_str[9:11])
137 mins =
int(dt_str[12:14])
138 secs =
int(dt_str[15:17])
139 dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
140 return dt_obj.strftime(
'%Y%j%H%M%S')
142 def _create_timestamp_using_mon_abbr(self, time_str):
144 Returns a properly formatted date/time stamp for MERIS L1B files from
145 an attribute in the file.
148 year =
int(time_str[7:11])
149 mon =
int(MONTH_ABBRS[time_str[3:6]])
150 dom =
int(time_str[0:2])
151 hrs =
int(time_str[12:14])
152 mins =
int(time_str[15:17])
153 secs =
int(time_str[18:20])
154 dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
155 return dt_obj.strftime(
'%Y%j%H%M%S')
157 def _create_timestamp_using_YMDTHMS(self, time_str):
159 Returns a properly formatted date/time stamp for MERIS L1B files from
160 an attribute in the file.
163 year =
int(time_str[0:4])
164 mon =
int(time_str[5:7])
165 dom =
int(time_str[8:10])
166 hrs =
int(time_str[11:13])
167 mins =
int(time_str[14:16])
168 secs =
int(time_str[17:19])
169 dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
170 return dt_obj.strftime(
'%Y%j%H%M%S')
172 def _extract_viirs_sdr(self, tar_path):
176 tar_file = tarfile.TarFile(tar_path)
177 tar_members = tar_file.getnames()
178 for mbr
in tar_members:
179 if mbr.startswith(
'SVM01'):
180 mbr_info = tar_file.getmember(mbr)
181 tar_file.extractall(members=[mbr_info], path=
'/tmp')
182 self.
file_path = os.path.join(
'/tmp', mbr)
186 exc_info = sys.exc_info()
187 for item
in exc_info:
193 def _get_data_from_anc_attributes(self):
195 Processes Ancillary data files.
197 instrument =
'Ancillary'
199 return file_type, instrument
201 def _get_data_from_l0_attributes(self):
203 Get the instrument and file type from the attributes for an L0 file.
205 file_type =
'unknown'
208 if title_parts[1].find(
'Level-0') != -1:
209 file_type =
'Level 0'
210 instrument = title_parts[0].strip()
211 return file_type, instrument
213 def _get_data_from_l1_attributes(self, title):
215 Get the instrument and file type from the attributes for an L1 file.
217 file_type =
'Level 1'
218 instrument =
'unknown'
219 possible_levels = {
'Level 1A':
'Level 1A',
'Level-1A':
'Level 1A',
220 'L1A':
'Level 1A',
'L1a':
'Level 1A',
221 'Level 1B':
'Level 1B',
'Level-1B':
'Level 1B',
222 'L1B':
'Level 1B',
'L1b':
'Level 1B'}
224 if title.find(
'Level 1') != -1:
225 working_title = title.replace(
'Level 1',
'Level-1')
227 working_title = title
228 working_title = re.sub(
"""^['"](.*)['"].*;$""",
'\g<1>', working_title)
229 title_parts = working_title.split()
230 for part
in title_parts:
231 if part
in KNOWN_SENSORS:
235 for ks
in KNOWN_SENSORS:
236 if part.find(ks) != -1:
240 if part
in possible_levels:
241 file_type = possible_levels[part]
242 if title.find(
'Browse Data') != -1:
243 file_type +=
' Browse Data'
244 return file_type, instrument
246 def _get_data_from_l2_attributes(self):
248 Get the instrument and file type from the attributes for an 20 file.
250 file_type =
'Level 2'
252 title_parts = self.
attributes[
'Title'].split()
253 if self.
attributes[
'Title'].find(
'Browse Data') != -1:
254 file_type +=
' Browse Data'
256 title_parts = self.
attributes[
'title'].split()
257 if self.
attributes[
'title'].find(
'Browse Data') != -1:
258 file_type +=
' Browse Data'
259 instrument = title_parts[0].strip()
260 return file_type, instrument
262 def _get_data_from_l3_attributes(self):
264 Get the instrument and file type from the attributes for an L3 file.
266 file_type =
'unknown'
273 if (working_title.find(
'Level-3 Binned') != -1)
or \
274 (working_title.find(
'level-3_binned') != -1):
275 file_type =
'Level 3 Binned'
276 elif working_title.find(
'Level-3 Standard Mapped Image') != -1:
277 file_type =
'Level 3 SMI'
279 return file_type, instrument
281 def _get_instrument(self):
283 Get the instrument from the attributes.
285 instrument =
'unknown'
288 title_parts = self.
attributes[
'Title'].split()
290 title_parts = self.
attributes[
'title'].split()
291 if title_parts[0].find(
'Level') == -1:
292 instrument = title_parts[0].strip()
295 if not (instrument
in KNOWN_SENSORS):
296 if instrument !=
'unknown':
298 instrument =
' '.join([instrument,
302 instrument = self.
attributes[
'Sensor Name'].strip()
304 (instrument
in KNOWN_SENSORS):
305 instrument = self.
attributes[
'Sensor'].strip()
308 def _get_l0_constructor_data(self):
310 Try to get data from the MODIS L0 constructor for this file.
312 l0cnst_results =
None
313 cmd = [
'l0cnst_write_modis',
'-n', self.
file_path]
315 l0_out = subprocess.Popen(cmd, shell=
False, stdout=subprocess.PIPE,
316 stderr=subprocess.PIPE).communicate()[0]
317 out_lines = l0_out.decode(
'utf-8').split(
'\n')
319 for line
in out_lines:
320 if line.find(
'Total packets') != -1:
322 packets_read =
float(parts[0])
323 if packets_read > 0.0:
328 if starttime
and stoptime:
329 l0cnst_results = {
'starttime' : starttime,
330 'stoptime' : stoptime}
332 l0cnst_results =
None
333 return l0cnst_results
335 def _get_l0_times(self):
337 Returns the start and end times for L0 data files.
339 if isinstance(self.
l0_data, dict):
340 start_time = self.
l0_data[
'starttime']
341 end_time = self.
l0_data[
'stoptime']
344 if line.find(
'starttime') != -1:
345 time_part = line.split(
'=')[1]
346 start_time = time_part[0:4] + time_part[5:7] + \
347 time_part[8:10] + time_part[11:13] + \
348 time_part[14:16] + time_part[17:19]
349 if line.find(
'stoptime') != -1:
350 time_part = line.split(
'=')[1]
351 end_time = time_part[0:4] + time_part[5:7] + \
352 time_part[8:10] + time_part[11:13] +\
353 time_part[14:16] + time_part[17:19]
354 return start_time, end_time
356 def _get_landsat_end_time(self, start_yr, start_doy, start_hr, start_min,
359 Compute and return the end time for a Landsat OLI data file, given the
360 start year, day of year, hour, mine, and second.
364 end_hr =
int(start_hr)
365 end_min =
int(start_min)
366 end_sec =
int(start_sec) + 1
383 end_time =
'{0:4d}{1:03d}{2:02d}{3:02d}{4:02d}'.format(end_yr,
390 def _get_landsat_times(self):
392 Computes and returns the start and end times for a Landsat OLI
398 acquisition_date = self.
attributes[
'DATE_ACQUIRED'].strip(
'"')
399 center_time = self.
attributes[
'SCENE_CENTER_TIME'].strip(
'"')
401 start_yr =
int(acquisition_date[0:4])
402 start_mon =
int(acquisition_date[5:7])
403 start_dom =
int(acquisition_date[8:10])
406 start_date_str =
'{0:04d}{1:03d}'.format(start_yr, start_doy)
407 start_hr = center_time[0:2]
408 start_min = center_time[3:5]
409 start_sec = center_time[6:8]
410 start_time =
''.join([start_date_str, start_hr, start_min,
413 start_hr, start_min, start_sec)
414 return start_time, end_time
416 def _get_time_from_coverage_field(self, coverage_time):
418 Returns the stamp computed from the coverage_time
420 if coverage_time[0:4] ==
'':
423 yr =
'{0:04d}'.format(
int(coverage_time[0:4]))
424 doy =
'{0:03d}'.format(
428 if coverage_time[11:13] ==
'':
431 hr =
'{0:02d}'.format(
int(coverage_time[11:13]))
432 if coverage_time[14:16] ==
'':
435 min =
'{0:02d}'.format(
int(coverage_time[14:16]))
436 if coverage_time[17:19] ==
'':
439 sec =
'{0:02d}'.format(
int(coverage_time[17:19]))
440 time_stamp =
''.join([yr,
str(doy), hr, min, sec])
446 API method to return the file's attributes.
452 Returns the start and end time for a file.
455 start_time =
'unable to determine start time'
456 end_time =
'unable to determine end time'
464 elif self.
file_type.find(
'Level 1') != -1:
466 elif self.
file_type.find(
'Level 2') != -1
or \
475 start_time = self.
attributes[
'Start Time'][0:13]
476 elif 'time_coverage_start' in self.
attributes:
500 start_time =
'{0:04}{1:03}{2}'.format(
506 end_time =
'{0:04}{1:03}{2}'.format(
512 elif self.
file_type.find(
'Level 0') != -1:
514 return start_time, end_time
516 def _get_type_using_platform(self):
517 levelMap = {
'L0':
'Level 0',
523 'L3 Binned':
'Level 3 Binned',
524 'L3 Mapped':
'Level 3 SMI'
526 instrumentMap = {
'SEAWIFS':
"SeaWiFS",
540 "AQUARIUS":
"Aquarius" ,
547 "HAWKEYE":
"HAWKEYE",
552 if self.
attributes[
'processing_level']
in levelMap:
555 if self.
attributes[
'title'].find(
'Geolocation') != -1:
558 if self.
attributes[
'title'].find(
'Standard Mapped Image') == -1:
561 if inst
in instrumentMap:
565 if inst.find(
'MODIS') != -1:
566 if plat.find(
'AQUA') != -1:
569 elif plat.find(
'TERRA') != -1:
572 elif inst.find(
'VIIRS') != -1:
573 if plat.find(
'SUOMI-NPP') != -1:
576 elif plat.find(
'JPSS-1') != -1:
579 elif inst.find(
'OLCI') != -1:
580 if plat ==
'SENTINEL-3' or plat.find(
'3A') != -1:
583 elif plat.find(
'3B') != -1:
586 elif inst.find(
'MSI') != -1:
587 if plat ==
'SENTINEL-2' or plat.find(
'2A') != -1:
590 elif plat.find(
'2B') != -1:
593 elif inst.find(
'OLI') != -1:
594 if plat ==
'LANDSAT-8':
597 elif plat ==
'LANDSAT-9':
602 def _get_file_type_from_attributes(self):
604 Determines the file type and instrument from the file attributes and
605 sets those values in the object.
614 elif 'ASSOCIATEDINSTRUMENTSHORTNAME' in self.
attributes:
617 if self.
attributes[
'PRODUCT'].find(
'MER_') != -1:
620 elif 'Instrument_Short_Name' in self.
attributes:
623 if self.
attributes[
'processing_level'].find(
'L1B') != -1:
630 elif mission ==
'JPSS-1':
638 if not self.
file_type.startswith(
'Level'):
641 if self.
attributes[
'Sensor'].find(
'SGLI') != -1:
649 Returns what type (L1A, L2, etc.) a file is and what
650 platform/sensor/instrument made the observations.
666 def _get_file_type_landsat(self):
668 Sets the file type and instrument for Landsat OLI data files
671 if self.
attributes[
'SENSOR_ID'].find(
'OLI_TIRS') != -1:
672 if self.
attributes[
'SPACECRAFT_ID'].find(
'LANDSAT_8') != -1:
680 if self.
attributes[
'DATA_TYPE'].find(
'L1G') != -1
or \
681 self.
attributes[
'DATA_TYPE'].find(
'L1GT') != -1
or \
682 self.
attributes[
'DATA_TYPE'].find(
'L1P') != -1
or \
683 self.
attributes[
'DATA_TYPE'].find(
'L1T') != -1:
686 if self.
attributes[
'PROCESSING_LEVEL'].find(
'L1G') != -1
or \
687 self.
attributes[
'PROCESSING_LEVEL'].find(
'L1GT') != -1
or \
688 self.
attributes[
'PROCESSING_LEVEL'].find(
'L1P') != -1
or \
689 self.
attributes[
'PROCESSING_LEVEL'].find(
'L1T') != -1
or\
690 self.
attributes[
'PROCESSING_LEVEL'].find(
'L1TP') != -1:
697 def _get_l0_start_stop_times(self, l0_lines):
699 Returns the start time and stop time if found in l0_lines.
703 for line
in l0_lines:
704 if line.find(
'starttime') != -1:
705 starttime = line.strip().split(
'=')[1]
706 if line.find(
'stoptime') != -1:
707 stoptime = line.strip().split(
'=')[1]
708 return starttime, stoptime
710 def _get_l1_goci_times(self):
712 Finds and returns timestamps for GOCI L1 data files.
719 elif 'processing start time' in self.
attributes:
721 mon = time.strptime(self.
attributes[
'processing start time']\
725 start_time =
'{0:04d}{1:03d}{2:02d}{3}{4}'.format(st_yr, doy,
727 self.
attributes[
'processing start time'][15:17],
728 self.
attributes[
'processing start time'][18:20])
733 elif 'processing end time' in self.
attributes:
735 mon = time.strptime(self.
attributes[
'processing end time']\
739 end_time =
'{0:04d}{1:03d}{2:02d}{3}{4}'.format(end_yr, doy,
741 self.
attributes[
'processing end time'][15:17],
742 self.
attributes[
'processing end time'][18:20])
743 return start_time, end_time
745 def _get_l1_hico_times(self):
747 Finds and returns timestamps for HICO L1 data files.
755 return start_time, end_time
757 def _get_l1_meris_times(self):
759 Finds and returns timestamps for MERIS L1 data files.
776 return start_time, end_time
778 def _get_l1_modis_times(self):
780 Finds and returns timestamps for MODIS L1 data files.
788 return start_time, end_time
790 def _get_l1_msi_times(self):
792 Finds and returns timestamps for MSI L1 data files.
797 end_time = start_time
798 return start_time, end_time
800 def _get_l1_ocm2_times(self):
802 Finds and returns timestamps for OCM2 L1 data files.
810 elif 'time_coverage_start' in self.
attributes:
821 return start_time, end_time
823 def _get_l1_octs_times(self):
825 Finds and returns timestamps for OCTS L1 data files.
838 self.
attributes[
'time_coverage_start'][11:13],
839 self.
attributes[
'time_coverage_start'][14:16],
851 return start_time, end_time
853 def _get_l1_sgli_times(self):
855 Finds and returns timestamps for SGLI L1 data files.
862 return start_time, end_time
864 def _get_l1_times(self):
866 Determines the times for Level 1 files.
878 start_time = self.
attributes[
'Start Time'][0:13]
879 elif 'time_coverage_start' in self.
attributes:
885 self.
attributes[
'time_coverage_start'][11:13],
886 self.
attributes[
'time_coverage_start'][14:16],
908 self.
attributes[
'time_coverage_start'][11:13],
909 self.
attributes[
'time_coverage_start'][14:16],
942 return start_time, end_time
944 def _get_type_using_l0_cnst(self):
946 Determines the type info based on results of l0cnst_write_modis.
949 if l0cnst_results
is not None:
953 if re.search(
r'^[aA]|^MOD00.P|^MYD\S+.A|^P1540064',
956 elif re.search(
r'^[tT]|^MOD\S+.A|^P0420064',
960 def _get_type_using_short_name(self):
962 Determines the type based on the short name.
965 if 'ASSOCIATEDPLATFORMSHORTNAME' in self.
attributes:
967 self.
attributes[
'ASSOCIATEDPLATFORMSHORTNAME']])
969 if self.
attributes[
'LONGNAME'].find(
'Geolocation') != -1:
971 elif self.
attributes[
'LONGNAME'].find(
'L1A') != -1:
973 elif self.
attributes[
'LONGNAME'].find(
'L1B') != -1:
976 def _get_type_using_title(self):
978 Determines the type based on the title field.
984 if title.find(
'OLCI Level 1b') != -1:
987 product_name = self.
attributes[
'product_name']
988 if '3A_OL_1_E' in product_name:
990 elif '3B_OL_1_E' in product_name:
992 elif (title.find(
'Level-1') != -1)
or (title.find(
'Level 1') != -1)
or \
993 (title.find(
'L1') != -1):
996 elif title.find(
'Level-2') != -1
or title.find(
'Level 2') != -1:
999 elif (title.find(
'Level-3') != -1)
or (title.find(
'level-3') != -1):
1002 elif title.find(
'Level-0') != -1:
1005 elif title.find(
'Ancillary') != -1:
1008 elif title.find(
'VIIRS') != -1:
1011 if self.
attributes[
'processing_level'].upper().find(
'L1A') != -1:
1013 elif self.
attributes[
'processing_level'].upper().find(
'L1B') != -1:
1015 elif self.
attributes[
'processing_level'].upper().find(
'GEO') != -1:
1017 elif title.find(
'L1A') != -1:
1019 elif title.find(
'L1B') != -1:
1022 def _read_metadata(self):
1024 Using the MetaUtils.readMetadata function, reads the metadata and loads
1025 it into the attributes member variable.
1031 if not (attrs
is None):