OB.DAAC Logo
NASA Logo
Ocean Color Science Software

ocssw V2022
get_obpg_file_type.py
Go to the documentation of this file.
1 
2 """
3 A class for determining the OBPG type of a file.
4 """
5 __version__ = '2.0'
6 
7 __author__ = 'melliott'
8 
9 import calendar
10 import datetime
11 import math
12 import optparse
13 import os
14 import re
15 import subprocess
16 import sys
17 import tarfile
18 import time
21 
22 
23 KNOWN_SENSORS = ['Aquarius', 'CZCS', 'HAWKEYE','HICO',
24  'HMODISA', 'HMODIST', 'MERIS', 'MODISA',
25  'MODIS Aqua', 'MODIST', 'MODIS Terra',
26  'MOS', 'OCIS', 'OCM2', 'OCTS', 'OSMI',
27  'SeaWiFS', 'SGLI','VIIRSN', 'VIIRSJ1']
28 
29 MONTH_ABBRS = dict((v.upper(), k) for k, v in enumerate(calendar.month_abbr))
30 
31 
33  """
34  Converts a number of milliseconds to a string representing the time of day.
35  """
36  secs = math.trunc(millisecs / 1000)
37  hrs = math.trunc(secs / 3600)
38  secs = secs - (hrs * 3600)
39  mins = math.trunc(secs / 60)
40  secs = secs - (mins * 60)
41  tm_str = '{0:02}{1:02}{2:02}'.format(hrs, mins, secs)
42  return tm_str
43 
44 def get_timestamp_from_month_day(sdate, stime):
45  """
46  Creates timestamp for VIIRS L1.
47  """
48  year = int(sdate[0:4])
49  mon = int(sdate[4:6])
50  dom = int(sdate[6:8])
51  hrs = int(stime[0:2])
52  mins = int(stime[2:4])
53  secs = int(stime[4:6])
54  dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
55  return dt_obj.strftime('%Y%j%H%M%S')
56 
57 def get_timestamp_from_year_day_mil(year, doy, millisecs):
58  """
59  Return a timestamp in the YYYYDDDHHMMSS form from a given year (yr), day
60  of year (doy) and milliseconds (after 00:00 - millisecs).
61  """
62  time_str = convert_millisecs_to_time_str(millisecs)
63  timestamp = '{0:04}{1:03}{2}'.format(year, doy, time_str)
64  return timestamp
65 
66 class ObpgFileTyper(object):
67  """
68  A class containing methods for finding the type of an OBPG file (e.g.
69  MODIS L2b, SeaWiFS L1A, Aquarius L3m, etc.
70  """
71 
72  def __init__(self, fpath):
73  """
74  Save the path of the file in question and set up default
75  values for several thing still to be found.
76  """
77  if os.path.exists(fpath):
78  if tarfile.is_tarfile(fpath):
79  # self.file_path = self._extract_viirs_sdr(fpath)
80  self._extract_viirs_sdr(fpath)
81  if not self.file_path:
82  err_msg = "Error! Cannot process file {0}.".format(fpath)
83  sys.exit(err_msg)
84  else:
85  self.file_path = fpath
86  self.file_type = 'unknown'
87  self.instrument = 'unknown'
88  self.start_time = 'unknown'
89  self.end_time = 'unknown'
90  self.attributes = None
91  self.l0_data = None
92  else:
93  err_msg = "Error! File {0} could not be found.".format(fpath)
94  sys.exit(err_msg)
95 
96 # def _convert_month_day_to_doy(self, sdate, stime):
97 # """
98 # Creates timestamp for VIIRS L1.
99 # """
100 # year = int(sdate[0:4])
101 # mon = int(sdate[4:6])
102 # dom = int(sdate[6:8])
103 # hrs = int(stime[0:2])
104 # mins = int(stime[2:4])
105 # secs = int(stime[4:6])
106 # dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
107 # return dt_obj.strftime('%Y%j%H%M%S')
108 
109  def _create_modis_l1_timestamp(self, rng_date, rng_time):
110  """
111  Returns a date/time stamp for a MODIS L1 file from the appropriate
112  RANGINGDATE and RANGINGTIME attributes. The returned date/time stamp
113  is of form YYYYDDDHHMMSS, where YYYY = year, DDD = day of year,
114  HH = hour, MM = minute, and SS = second.
115  """
116  year = int(rng_date[0:4])
117  mon = int(rng_date[5:7])
118  dom = int(rng_date[8:10])
119  hrs = int(rng_time[0:2])
120  mins = int(rng_time[3:5])
121  secs = int(rng_time[6:8])
122  milsecs = int(rng_time[9:15])
123  dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs, milsecs)
124  if milsecs >= 500000:
125  dt_delta = datetime.timedelta(seconds=1)
126  dt_obj += dt_delta
127  return dt_obj.strftime('%Y%j%H%M%S')
128 
129  def _create_octs_l1_timestamp(self, dt_str):
130  """
131  Creates a timestamp for an OCTS L1.
132  """
133  year = int(dt_str[0:4])
134  mon = int(dt_str[4:6])
135  dom = int(dt_str[6:8])
136  hrs = int(dt_str[9:11])
137  mins = int(dt_str[12:14])
138  secs = int(dt_str[15:17])
139  dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
140  return dt_obj.strftime('%Y%j%H%M%S')
141 
142  def _create_timestamp_using_mon_abbr(self, time_str):
143  """
144  Returns a properly formatted date/time stamp for MERIS L1B files from
145  an attribute in the file.
146  """
147  # Todo: Check that MERIS' and Python's month abbreviations match up ...
148  year = int(time_str[7:11])
149  mon = int(MONTH_ABBRS[time_str[3:6]])
150  dom = int(time_str[0:2])
151  hrs = int(time_str[12:14])
152  mins = int(time_str[15:17])
153  secs = int(time_str[18:20])
154  dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
155  return dt_obj.strftime('%Y%j%H%M%S')
156 
157  def _create_timestamp_using_YMDTHMS(self, time_str):
158  """
159  Returns a properly formatted date/time stamp for MERIS L1B files from
160  an attribute in the file.
161  """
162  # Todo: Check that MERIS' and Python's month abbreviations match up ...
163  year = int(time_str[0:4])
164  mon = int(time_str[5:7])
165  dom = int(time_str[8:10])
166  hrs = int(time_str[11:13])
167  mins = int(time_str[14:16])
168  secs = int(time_str[17:19])
169  dt_obj = datetime.datetime(year, mon, dom, hrs, mins, secs)
170  return dt_obj.strftime('%Y%j%H%M%S')
171 
172  def _extract_viirs_sdr(self, tar_path):
173  self.file_path = None
174  try:
175  # orig_path = os.getcwd()
176  tar_file = tarfile.TarFile(tar_path)
177  tar_members = tar_file.getnames()
178  for mbr in tar_members:
179  if mbr.startswith('SVM01'):
180  mbr_info = tar_file.getmember(mbr)
181  tar_file.extractall(members=[mbr_info], path='/tmp')
182  self.file_path = os.path.join('/tmp', mbr)
183  break
184  tar_file.close()
185  except:
186  exc_info = sys.exc_info()
187  for item in exc_info:
188  print(str(item))
189 
190  def get_attributes(self):
191  return self.attributes
192 
193  def _get_data_from_anc_attributes(self):
194  """
195  Processes Ancillary data files.
196  """
197  instrument = 'Ancillary'
198  file_type = self.attributes['Data Type']
199  return file_type, instrument
200 
201  def _get_data_from_l0_attributes(self):
202  """
203  Get the instrument and file type from the attributes for an L0 file.
204  """
205  file_type = 'unknown'
206  # instrument = 'unknown'
207  title_parts = self.attributes['Title'].split()
208  if title_parts[1].find('Level-0') != -1:
209  file_type = 'Level 0'
210  instrument = title_parts[0].strip()
211  return file_type, instrument
212 
213  def _get_data_from_l1_attributes(self, title):
214  """
215  Get the instrument and file type from the attributes for an L1 file.
216  """
217  file_type = 'Level 1'
218  instrument = 'unknown'
219  possible_levels = {'Level 1A': 'Level 1A', 'Level-1A': 'Level 1A',
220  'L1A': 'Level 1A', 'L1a': 'Level 1A',
221  'Level 1B': 'Level 1B', 'Level-1B': 'Level 1B',
222  'L1B': 'Level 1B', 'L1b': 'Level 1B'}
223 # ks = known_sensors
224  if title.find('Level 1') != -1:
225  working_title = title.replace('Level 1', 'Level-1')
226  else:
227  working_title = title
228  working_title = re.sub("""^['"](.*)['"].*;$""", '\g<1>', working_title)
229  title_parts = working_title.split()
230  for part in title_parts:
231  if part in KNOWN_SENSORS:
232  instrument = part
233  else:
234  # found = False
235  for ks in KNOWN_SENSORS:
236  if part.find(ks) != -1:
237  instrument = ks
238  # found = True
239  break
240  if part in possible_levels:
241  file_type = possible_levels[part]
242  if title.find('Browse Data') != -1:
243  file_type += ' Browse Data'
244  return file_type, instrument
245 
246  def _get_data_from_l2_attributes(self):
247  """
248  Get the instrument and file type from the attributes for an 20 file.
249  """
250  file_type = 'Level 2'
251  if 'Title' in self.attributes:
252  title_parts = self.attributes['Title'].split()
253  if self.attributes['Title'].find('Browse Data') != -1:
254  file_type += ' Browse Data'
255  elif 'title' in self.attributes:
256  title_parts = self.attributes['title'].split()
257  if self.attributes['title'].find('Browse Data') != -1:
258  file_type += ' Browse Data'
259  instrument = title_parts[0].strip()
260  return file_type, instrument
261 
262  def _get_data_from_l3_attributes(self):
263  """
264  Get the instrument and file type from the attributes for an L3 file.
265  """
266  file_type = 'unknown'
267  if 'Title' in self.attributes:
268  working_title = self.attributes['Title']
269  elif 'title' in self.attributes:
270  working_title = self.attributes['title']
271  elif 'instrument' in self.attributes:
272  pass
273  if (working_title.find('Level-3 Binned') != -1) or \
274  (working_title.find('level-3_binned') != -1):
275  file_type = 'Level 3 Binned'
276  elif working_title.find('Level-3 Standard Mapped Image') != -1:
277  file_type = 'Level 3 SMI'
278  instrument = self._get_instrument()
279  return file_type, instrument
280 
281  def _get_instrument(self):
282  """
283  Get the instrument from the attributes.
284  """
285  instrument = 'unknown'
286  if 'Title' in self.attributes or 'title' in self.attributes:
287  if 'Title' in self.attributes:
288  title_parts = self.attributes['Title'].split()
289  else:
290  title_parts = self.attributes['title'].split()
291  if title_parts[0].find('Level') == -1:
292  instrument = title_parts[0].strip()
293  elif 'instrument' in self.attributes:
294  instrument = self.attributes['instrument']
295  if not (instrument in KNOWN_SENSORS):
296  if instrument != 'unknown':
297  if 'platform' in self.attributes:
298  instrument = ' '.join([instrument,
299  self.attributes['platform']])
300  else:
301  if 'Sensor Name' in self.attributes:
302  instrument = self.attributes['Sensor Name'].strip()
303  if ('Sensor' in self.attributes) and not \
304  (instrument in KNOWN_SENSORS):
305  instrument = self.attributes['Sensor'].strip()
306  return instrument
307 
308  def _get_l0_constructor_data(self):
309  """
310  Try to get data from the MODIS L0 constructor for this file.
311  """
312  l0cnst_results = None
313  cmd = ['l0cnst_write_modis', '-n', self.file_path]
314  try:
315  l0_out = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE,
316  stderr=subprocess.PIPE).communicate()[0]
317  out_lines = l0_out.decode('utf-8').split('\n')
318  is_l0 = False
319  for line in out_lines:
320  if line.find('Total packets') != -1:
321  parts = line.split()
322  packets_read = float(parts[0])
323  if packets_read > 0.0:
324  is_l0 = True
325  break
326  if is_l0:
327  starttime, stoptime = self._get_l0_start_stop_times(out_lines)
328  if starttime and stoptime:
329  l0cnst_results = {'starttime' : starttime,
330  'stoptime' : stoptime}
331  except Exception:
332  l0cnst_results = None
333  return l0cnst_results
334 
335  def _get_l0_times(self):
336  """
337  Returns the start and end times for L0 data files.
338  """
339  if isinstance(self.l0_data, dict):
340  start_time = self.l0_data['starttime']
341  end_time = self.l0_data['stoptime']
342  else:
343  for line in self.l0_data:
344  if line.find('starttime') != -1:
345  time_part = line.split('=')[1]
346  start_time = time_part[0:4] + time_part[5:7] + \
347  time_part[8:10] + time_part[11:13] + \
348  time_part[14:16] + time_part[17:19]
349  if line.find('stoptime') != -1:
350  time_part = line.split('=')[1]
351  end_time = time_part[0:4] + time_part[5:7] + \
352  time_part[8:10] + time_part[11:13] +\
353  time_part[14:16] + time_part[17:19]
354  return start_time, end_time
355 
356  def _get_landsat_end_time(self, start_yr, start_doy, start_hr, start_min,
357  start_sec):
358  """
359  Compute and return the end time for a Landsat OLI data file, given the
360  start year, day of year, hour, mine, and second.
361  """
362  end_yr = start_yr
363  end_doy = start_doy
364  end_hr = int(start_hr)
365  end_min = int(start_min)
366  end_sec = int(start_sec) + 1
367  if end_sec >= 60:
368  end_sec = 0
369  end_min += 1
370  if end_min >= 60:
371  end_min = 0
372  end_hr += 1
373  if end_hr >= 24:
374  end_hr = 0
375  end_doy += 1
376  if end_yr % 4 == 0:
377  if end_doy > 366:
378  end_doy = 1
379  end_yr += 1
380  elif end_doy > 365:
381  end_doy = 1
382  end_yr += 1
383  end_time = '{0:4d}{1:03d}{2:02d}{3:02d}{4:02d}'.format(end_yr,
384  end_doy,
385  end_hr,
386  end_min,
387  end_sec)
388  return end_time
389 
390  def _get_landsat_times(self):
391  """
392  Computes and returns the start and end times for a Landsat OLI
393  data file.
394  """
395  # The Landsat OLI metadata only contains an acquisition date and
396  # scene center time. (For now) It is assumed that the scene center
397  # time is the start time and the end time is one second later.
398  acquisition_date = self.attributes['DATE_ACQUIRED'].strip('"')
399  center_time = self.attributes['SCENE_CENTER_TIME'].strip('"')
400 
401  start_yr = int(acquisition_date[0:4])
402  start_mon = int(acquisition_date[5:7])
403  start_dom = int(acquisition_date[8:10])
404  start_doy =seadasutils.time_utils.convert_month_day_to_doy(start_mon, start_dom,
405  start_yr)
406  start_date_str = '{0:04d}{1:03d}'.format(start_yr, start_doy)
407  start_hr = center_time[0:2]
408  start_min = center_time[3:5]
409  start_sec = center_time[6:8]
410  start_time = ''.join([start_date_str, start_hr, start_min,
411  start_sec])
412  end_time = self._get_landsat_end_time(start_yr, start_doy,
413  start_hr, start_min, start_sec)
414  return start_time, end_time
415 
416  def _get_time_from_coverage_field(self, coverage_time):
417  """
418  Returns the stamp computed from the coverage_time
419  """
420  if coverage_time[0:4] == '':
421  yr = '0000'
422  else:
423  yr = '{0:04d}'.format(int(coverage_time[0:4]))
424  doy = '{0:03d}'.format(
426  coverage_time[8:10],
427  coverage_time[0:4]))
428  if coverage_time[11:13] == '':
429  hr = '00'
430  else:
431  hr = '{0:02d}'.format(int(coverage_time[11:13]))
432  if coverage_time[14:16] == '':
433  min = '00'
434  else:
435  min = '{0:02d}'.format(int(coverage_time[14:16]))
436  if coverage_time[17:19] == '':
437  sec = '00'
438  else:
439  sec = '{0:02d}'.format(int(coverage_time[17:19]))
440  time_stamp = ''.join([yr, str(doy), hr, min, sec])
441  return time_stamp
442 
443 
445  """
446  API method to return the file's attributes.
447  """
448  return self.attributes
449 
450  def get_file_times(self):
451  """
452  Returns the start and end time for a file.
453  """
454 # print 'self.instrument: "' + self.instrument + '"'
455  start_time = 'unable to determine start time'
456  end_time = 'unable to determine end time'
457  if 'VIIRS' in self.instrument and self.file_type == 'SDR':
458  start_time = get_timestamp_from_month_day(
459  self.attributes['Beginning_Date'],
460  self.attributes['Beginning_Time'])
461  end_time = get_timestamp_from_month_day(
462  self.attributes['Ending_Date'],
463  self.attributes['Ending_Time'])
464  elif self.file_type.find('Level 1') != -1:
465  start_time, end_time = self._get_l1_times()
466  elif self.file_type.find('Level 2') != -1 or \
467  self.file_type.find('Level 3') != -1:
468  try:
469  if 'time_coverage_start' in self.attributes:
470  start_time = self._get_time_from_coverage_field(self.attributes['time_coverage_start'])
471  end_time = self._get_time_from_coverage_field(self.attributes['time_coverage_end'])
472  elif 'RANGEBEGINNINGDATE' in self.attributes:
473  start_time, end_time = self._get_l1_modis_times()
474  elif 'Start Time' in self.attributes:
475  start_time = self.attributes['Start Time'][0:13]
476  elif 'time_coverage_start' in self.attributes:
477  start_time = self._get_time_from_coverage_field(
478  self.attributes['time_coverage_start'])
479  elif 'Start Day' in self.attributes:
480  start_time = get_timestamp_from_year_day_mil(
481  int(self.attributes['Start Year']),
482  int(self.attributes['Start Day']),
483  int(self.attributes['Start Millisec'])
484  )
485  if 'End Time' in self.attributes:
486  end_time = self.attributes['End Time'][0:13]
487  elif 'time_coverage_end' in self.attributes:
488  end_time = self._get_time_from_coverage_field(
489  self.attributes['time_coverage_end'])
490  elif 'End Day' in self.attributes:
492  int(self.attributes['End Year']),
493  int(self.attributes['End Day']),
494  int(self.attributes['End Millisec'])
495  )
496  except KeyError:
497  if self.instrument.find('Aquarius') != -1:
498  stime_str = convert_millisecs_to_time_str(
499  int(self.attributes['Start Millisec']))
500  start_time = '{0:04}{1:03}{2}'.format(
501  int(self.attributes['Start Year']),
502  int(self.attributes['Start Day']),
503  stime_str)
504  etime_str = convert_millisecs_to_time_str(
505  int(self.attributes['Start Millisec']))
506  end_time = '{0:04}{1:03}{2}'.format(
507  int(self.attributes['End Year']),
508  int(self.attributes['End Day']),
509  etime_str)
510  else:
511  raise
512  elif self.file_type.find('Level 0') != -1:
513  start_time, end_time = self._get_l0_times()
514  return start_time, end_time
515 
516  def _get_type_using_platform(self):
517  levelMap = {'L0':'Level 0',
518  'L1A':'Level 1A',
519  'L1C':'Level 1C',
520  'GEO':'GEO',
521  'L1B':'Level 1B',
522  'L2':'Level 2',
523  'L3 Binned':'Level 3 Binned',
524  'L3 Mapped':'Level 3 SMI'
525  }
526  instrumentMap = {'SEAWIFS': "SeaWiFS",
527  "MOS": "MOS",
528  "OCTS": "OCTS",
529  "AVHRR":"AVHRR",
530  "OSMI": "OSMI",
531  "CZCS": "CZCS",
532  "OCM1": "OCM1",
533  "OCM2": "OCM2",
534  "OCM-2": "OCM2",
535  "MERIS":"MERIS",
536  "OCRVC":"OCRVC",
537  "HICO": "HICO",
538  "GOCI": "GOCI",
539  # "OLI": "OLI",
540  "AQUARIUS": "Aquarius" ,
541  "OCIA": "OCIA",
542  "AVIRIS": "AVIRIS",
543  "PRISM": "PRISM",
544  "SGLI": "SGLI",
545  "L5TM": "L5TM",
546  "L7ETM": "L7ETM",
547  "HAWKEYE": "HAWKEYE",
548  "MISR": "MISR",
549  "OCI": "OCI"
550  }
551 
552  if self.attributes['processing_level'] in levelMap:
553  self.file_type = levelMap[self.attributes['processing_level']]
554  if 'title' in self.attributes:
555  if self.attributes['title'].find('Geolocation') != -1:
556  self.file_type = 'GEO'
557  if self.file_type == 'Level 3 SMI':
558  if self.attributes['title'].find('Standard Mapped Image') == -1:
559  self.file_type == 'Level 3 Mapped'
560  inst = self.attributes['instrument'].upper()
561  if inst in instrumentMap:
562  self.instrument = instrumentMap[inst]
563  return True
564  plat = self.attributes['platform'].upper()
565  if inst.find('MODIS') != -1:
566  if plat.find('AQUA') != -1:
567  self.instrument = "MODIS Aqua"
568  return True
569  elif plat.find('TERRA') != -1:
570  self.instrument = "MODIS Terra"
571  return True
572  elif inst.find('VIIRS') != -1:
573  if plat.find('SUOMI-NPP') != -1:
574  self.instrument = "VIIRS NPP"
575  return True
576  elif plat.find('JPSS-1') != -1:
577  self.instrument = "VIIRS J1"
578  return True
579  elif inst.find('OLCI') != -1:
580  if plat == 'SENTINEL-3' or plat.find('3A') != -1:
581  self.instrument = "OLCI S3A"
582  return True
583  elif plat.find('3B') != -1:
584  self.instrument = "OLCI S3B"
585  return True
586  elif inst.find('MSI') != -1:
587  if plat == 'SENTINEL-2' or plat.find('2A') != -1:
588  self.instrument = "MSI S2A"
589  return True
590  elif plat.find('2B') != -1:
591  self.instrument = "MSI S2B"
592  return True
593  elif inst.find('OLI') != -1:
594  if plat == 'LANDSAT-8':
595  self.instrument = "OLI L8"
596  return True
597  elif plat == 'LANDSAT-9':
598  self.instrument = "OLI L9"
599  return True
600  return False
601 
602  def _get_file_type_from_attributes(self):
603  """
604  Determines the file type and instrument from the file attributes and
605  sets those values in the object.
606  """
607 
608  if ('platform' in self.attributes) and ('instrument' in self.attributes) and ('processing_level' in self.attributes):
609  if self._get_type_using_platform():
610  return
611 
612  if 'Title' in self.attributes or 'title' in self.attributes:
613  self._get_type_using_title()
614  elif 'ASSOCIATEDINSTRUMENTSHORTNAME' in self.attributes:
616  elif 'PRODUCT' in self.attributes:
617  if self.attributes['PRODUCT'].find('MER_') != -1:
618  self.instrument = 'MERIS'
619  self.file_type = 'Level 1B'
620  elif 'Instrument_Short_Name' in self.attributes:
621  self.instrument = self.attributes['Instrument_Short_Name'].upper()
622  if self.instrument == 'HICO':
623  if self.attributes['processing_level'].find('L1B') != -1:
624  self.file_type = 'Level 1B'
625  if self.instrument == 'VIIRS':
626  if 'Mission_Name' in self.attributes:
627  mission = self.attributes['Mission_Name']
628  if mission == 'NPP':
629  self.instrument = 'VIIRS NPP'
630  elif mission == 'JPSS-1':
631  self.instrument = 'VIIRS J1'
632  if 'N_Dataset_Type_Tag' in self.attributes:
633  self.file_type = self.attributes['N_Dataset_Type_Tag']
634  elif 'Product Level' in self.attributes:
635  if 'Sensor name' in self.attributes:
636  self.instrument = self.attributes['Sensor name']
637  self.file_type = self.attributes['Product Level']
638  if not self.file_type.startswith('Level'):
639  self.file_type = 'Level ' + self.file_type
640  elif 'Sensor' in self.attributes:
641  if self.attributes['Sensor'].find('SGLI') != -1:
642  self.instrument = 'SGLI'
643  self.file_type = 'Level 1B'
644  elif 'SENSOR_ID' in self.attributes:
646 
647  def get_file_type(self):
648  """
649  Returns what type (L1A, L2, etc.) a file is and what
650  platform/sensor/instrument made the observations.
651  """
652  orig_path = None
653  self._read_metadata()
654  if self.attributes:
656  else:
658  if self.instrument.find('MODISA') != -1:
659  self.instrument = 'MODIS Aqua'
660  if self.instrument.find('MODIST') != -1:
661  self.instrument = 'MODIS Terra'
662  if orig_path:
663  self.file_path = orig_path
664  return self.file_type, self.instrument
665 
666  def _get_file_type_landsat(self):
667  """
668  Sets the file type and instrument for Landsat OLI data files
669  """
670  # Landsat 8 OLI
671  if self.attributes['SENSOR_ID'].find('OLI_TIRS') != -1:
672  if self.attributes['SPACECRAFT_ID'].find('LANDSAT_8') != -1:
673  self.instrument = 'OLI L8'
674  else:
675  self.instrument = 'OLI L9'
676  else:
677  self.instrument = self.attributes['SENSOR_ID'].\
678  strip().strip('"')
679  if 'DATA_TYPE' in self.attributes:
680  if self.attributes['DATA_TYPE'].find('L1G') != -1 or \
681  self.attributes['DATA_TYPE'].find('L1GT') != -1 or \
682  self.attributes['DATA_TYPE'].find('L1P') != -1 or \
683  self.attributes['DATA_TYPE'].find('L1T') != -1:
684  self.file_type = 'Level 1B'
685  elif 'PROCESSING_LEVEL' in self.attributes:
686  if self.attributes['PROCESSING_LEVEL'].find('L1G') != -1 or \
687  self.attributes['PROCESSING_LEVEL'].find('L1GT') != -1 or \
688  self.attributes['PROCESSING_LEVEL'].find('L1P') != -1 or \
689  self.attributes['PROCESSING_LEVEL'].find('L1T') != -1 or\
690  self.attributes['PROCESSING_LEVEL'].find('L1TP') != -1:
691  self.file_type = 'Level 1B'
692  else:
693  self.file_type = 'Level ' + self.attributes['DATA_TYPE'].\
694  strip().strip('"')
695 
696 
697  def _get_l0_start_stop_times(self, l0_lines):
698  """
699  Returns the start time and stop time if found in l0_lines.
700  """
701  starttime = None
702  stoptime = None
703  for line in l0_lines:
704  if line.find('starttime') != -1:
705  starttime = line.strip().split('=')[1]
706  if line.find('stoptime') != -1:
707  stoptime = line.strip().split('=')[1]
708  return starttime, stoptime
709 
710  def _get_l1_goci_times(self):
711  """
712  Finds and returns timestamps for GOCI L1 data files.
713  """
714  if 'Scene Start time' in self.attributes and self.attributes['Scene Start time'] != 'unknown':
715  # start_time = self.attributes['Scene Start time']
716  start_time = self._create_timestamp_using_mon_abbr(self.attributes['Scene Start time'])
717  elif 'start_time' in self.attributes and self.attributes['start_time'] != 'unknown':
718  start_time = self.attributes['start_time']
719  elif 'processing start time' in self.attributes:
720  st_yr = int(self.attributes['processing start time'][7:11])
721  mon = time.strptime(self.attributes['processing start time']\
722  [3:6], '%b').tm_mon
723  dom = int(self.attributes['processing start time'][0:2])
725  start_time = '{0:04d}{1:03d}{2:02d}{3}{4}'.format(st_yr, doy,
726  int(self.attributes['processing start time'][12:14]),
727  self.attributes['processing start time'][15:17],
728  self.attributes['processing start time'][18:20])
729  if 'Scene end time' in self.attributes and self.attributes['Scene end time'] != 'unknown':
730  end_time = self._create_timestamp_using_mon_abbr(self.attributes['Scene end time'])
731  elif 'end_time' in self.attributes and self.attributes['end_time'] != 'unknown':
732  end_time = self.attributes['end_time']
733  elif 'processing end time' in self.attributes:
734  end_yr = int(self.attributes['processing end time'][7:11])
735  mon = time.strptime(self.attributes['processing end time']\
736  [3:6], '%b').tm_mon
737  dom = int(self.attributes['processing end time'][0:2])
739  end_time = '{0:04d}{1:03d}{2:02d}{3}{4}'.format(end_yr, doy,
740  int(self.attributes['processing end time'][12:14]),
741  self.attributes['processing end time'][15:17],
742  self.attributes['processing end time'][18:20])
743  return start_time, end_time
744 
745  def _get_l1_hico_times(self):
746  """
747  Finds and returns timestamps for HICO L1 data files.
748  """
749  start_time = get_timestamp_from_month_day(\
750  self.attributes['Beginning_Date'],
751  self.attributes['Beginning_Time'])
752  end_time = get_timestamp_from_month_day(
753  self.attributes['Ending_Date'],
754  self.attributes['Ending_Time'])
755  return start_time, end_time
756 
757  def _get_l1_meris_times(self):
758  """
759  Finds and returns timestamps for MERIS L1 data files.
760  """
761  if 'FIRST_LINE_TIME' in self.attributes:
762  start_time = self._create_timestamp_using_mon_abbr(
763  self.attributes['FIRST_LINE_TIME'])
764  end_time = self._create_timestamp_using_mon_abbr(
765  self.attributes['LAST_LINE_TIME'])
766  elif 'startTime' in self.attributes:
767  start_time = self._create_timestamp_using_YMDTHMS(
768  self.attributes['startTime'])
769  end_time = self._create_timestamp_using_YMDTHMS(
770  self.attributes['stopTime'])
771  else:
772  start_time = self._create_timestamp_using_mon_abbr(
773  self.attributes['start_date'].strip('"'))
774  end_time = self._create_timestamp_using_mon_abbr(
775  self.attributes['stop_date'].strip('"'))
776  return start_time, end_time
777 
778  def _get_l1_modis_times(self):
779  """
780  Finds and returns timestamps for MODIS L1 data files.
781  """
782  start_time = self._create_modis_l1_timestamp(
783  self.attributes['RANGEBEGINNINGDATE'],
784  self.attributes['RANGEBEGINNINGTIME'])
785  end_time = self._create_modis_l1_timestamp(
786  self.attributes['RANGEENDINGDATE'],
787  self.attributes['RANGEENDINGTIME'])
788  return start_time, end_time
789 
790  def _get_l1_msi_times(self):
791  """
792  Finds and returns timestamps for MSI L1 data files.
793  """
794  if 'startTime' in self.attributes:
795  start_time = self._create_timestamp_using_YMDTHMS(
796  self.attributes['startTime'])
797  end_time = start_time
798  return start_time, end_time
799 
800  def _get_l1_ocm2_times(self):
801  """
802  Finds and returns timestamps for OCM2 L1 data files.
803  """
804  #yr, doy, millisecs
805  if 'Start Year' in self.attributes:
806  start_time = get_timestamp_from_year_day_mil(\
807  self.attributes['Start Year'],
808  self.attributes['Start Day'],
809  self.attributes['Start Millisec'])
810  elif 'time_coverage_start' in self.attributes:
811  start_time = self._get_time_from_coverage_field(
812  self.attributes['time_coverage_start'])
813  if 'End Year' in self.attributes:
815  self.attributes['End Year'],
816  self.attributes['End Day'],
817  self.attributes['End Millisec'])
818  elif 'time_coverage_end' in self.attributes:
819  end_time = self._get_time_from_coverage_field(
820  self.attributes['time_coverage_end'])
821  return start_time, end_time
822 
823  def _get_l1_octs_times(self):
824  """
825  Finds and returns timestamps for OCTS L1 data files.
826  """
827  if 'Start Time' in self.attributes:
828  start_time = self._create_octs_l1_timestamp(
829  self.attributes['Start Time'])
830  end_time = self._create_octs_l1_timestamp(
831  self.attributes['End Time'])
832  else:
833  start_time = get_timestamp_from_month_day(''.join([
834  self.attributes['time_coverage_start'][0:4],
835  self.attributes['time_coverage_start'][5:7],
836  self.attributes['time_coverage_start'][8:10]
837  ]), ''.join([
838  self.attributes['time_coverage_start'][11:13],
839  self.attributes['time_coverage_start'][14:16],
840  self.attributes['time_coverage_start'][17:19]
841  ]))
842  end_time = get_timestamp_from_month_day(''.join([
843  self.attributes['time_coverage_end'][0:4],
844  self.attributes['time_coverage_end'][5:7],
845  self.attributes['time_coverage_end'][8:10]
846  ]), ''.join([
847  self.attributes['time_coverage_end'][11:13],
848  self.attributes['time_coverage_end'][14:16],
849  self.attributes['time_coverage_end'][17:19]
850  ]))
851  return start_time, end_time
852 
853  def _get_l1_sgli_times(self):
854  """
855  Finds and returns timestamps for SGLI L1 data files.
856  """
857  if 'Scene_start_time' in self.attributes:
858  start_time = self._create_octs_l1_timestamp(
859  self.attributes['Scene_start_time'])
860  end_time = self._create_octs_l1_timestamp(
861  self.attributes['Scene_end_time'])
862  return start_time, end_time
863 
864  def _get_l1_times(self):
865  """
866  Determines the times for Level 1 files.
867  """
868  start_time = None
869  end_time = None
870  if self.instrument.find('SeaWiFS')!= -1 or \
871  self.instrument.find('Aquarius') != -1 or \
872  self.instrument.find('CZCS') != -1 or \
873  self.instrument.find('MOS') != -1 or \
874  self.instrument.find('OCIS') != -1 or \
875  self.instrument.find('OSMI') != -1 or \
876  self.instrument.find('VIIRS') != -1:
877  if 'Start Time' in self.attributes:
878  start_time = self.attributes['Start Time'][0:13]
879  elif 'time_coverage_start' in self.attributes:
880  start_time = get_timestamp_from_month_day(''.join([
881  self.attributes['time_coverage_start'][0:4],
882  self.attributes['time_coverage_start'][5:7],
883  self.attributes['time_coverage_start'][8:10]
884  ]), ''.join([
885  self.attributes['time_coverage_start'][11:13],
886  self.attributes['time_coverage_start'][14:16],
887  self.attributes['time_coverage_start'][17:19]
888  ]))
889  if 'End Time' in self.attributes:
890  end_time = self.attributes['End Time'][0:13]
891  elif 'time_coverage_end' in self.attributes:
892  end_time = get_timestamp_from_month_day(''.join([
893  self.attributes['time_coverage_end'][0:4],
894  self.attributes['time_coverage_end'][5:7],
895  self.attributes['time_coverage_end'][8:10]
896  ]), ''.join([
897  self.attributes['time_coverage_end'][11:13],
898  self.attributes['time_coverage_end'][14:16],
899  self.attributes['time_coverage_end'][17:19]
900  ]))
901  elif self.instrument.find('HAWKEYE') != -1:
902  if 'time_coverage_start' in self.attributes:
903  start_time = get_timestamp_from_month_day(''.join([
904  self.attributes['time_coverage_start'][0:4],
905  self.attributes['time_coverage_start'][5:7],
906  self.attributes['time_coverage_start'][8:10]
907  ]), ''.join([
908  self.attributes['time_coverage_start'][11:13],
909  self.attributes['time_coverage_start'][14:16],
910  str(round(float(self.attributes['time_coverage_start'][17:23])))
911  ]))
912  if 'time_coverage_end' in self.attributes:
913  end_time = get_timestamp_from_month_day(''.join([
914  self.attributes['time_coverage_end'][0:4],
915  self.attributes['time_coverage_end'][5:7],
916  self.attributes['time_coverage_end'][8:10]
917  ]), ''.join([
918  self.attributes['time_coverage_end'][11:13],
919  self.attributes['time_coverage_end'][14:16],
920  str(round(float(self.attributes['time_coverage_start'][17:23])))
921  ]))
922  elif self.instrument.find('MODIS') != -1:
923  start_time, end_time = self._get_l1_modis_times()
924  elif self.instrument.find('OCTS') != -1:
925  start_time, end_time = self._get_l1_octs_times()
926  elif self.instrument.find('MERIS') != -1:
927  start_time, end_time = self._get_l1_meris_times()
928  elif self.instrument.find('OLCI') != -1:
929  start_time, end_time = self._get_l1_meris_times()
930  elif self.instrument.find('OCM2') != -1:
931  start_time, end_time = self._get_l1_ocm2_times()
932  elif self.instrument.find('HICO') != -1:
933  start_time, end_time = self._get_l1_hico_times()
934  elif self.instrument.find('GOCI') != -1:
935  start_time, end_time = self._get_l1_goci_times()
936  elif self.instrument.find('MSI') != -1:
937  start_time, end_time = self._get_l1_msi_times()
938  elif self.instrument.find('OLI') != -1:
939  start_time, end_time = self._get_landsat_times()
940  elif self.instrument.find('SGLI') != -1:
941  start_time, end_time = self._get_l1_sgli_times()
942  return start_time, end_time
943 
944  def _get_type_using_l0_cnst(self):
945  """
946  Determines the type info based on results of l0cnst_write_modis.
947  """
948  l0cnst_results = self._get_l0_constructor_data()
949  if l0cnst_results is not None:
950  self.l0_data = l0cnst_results
951  self.file_type = 'Level 0'
952  self.instrument = 'MODIS'
953  if re.search(r'^[aA]|^MOD00.P|^MYD\S+.A|^P1540064',
954  os.path.basename(self.file_path)):
955  self.instrument = ' '.join([self.instrument, 'Aqua'])
956  elif re.search(r'^[tT]|^MOD\S+.A|^P0420064',
957  os.path.basename(self.file_path)):
958  self.instrument = ' '.join([self.instrument, 'Terra'])
959 
960  def _get_type_using_short_name(self):
961  """
962  Determines the type based on the short name.
963  """
964  self.instrument = self.attributes['ASSOCIATEDINSTRUMENTSHORTNAME']
965  if 'ASSOCIATEDPLATFORMSHORTNAME' in self.attributes:
966  self.instrument = ' '.join([self.instrument,
967  self.attributes['ASSOCIATEDPLATFORMSHORTNAME']])
968  if 'LONGNAME' in self.attributes:
969  if self.attributes['LONGNAME'].find('Geolocation') != -1:
970  self.file_type = 'GEO'
971  elif self.attributes['LONGNAME'].find('L1A') != -1:
972  self.file_type = 'Level 1A'
973  elif self.attributes['LONGNAME'].find('L1B') != -1:
974  self.file_type = 'Level 1B'
975 
976  def _get_type_using_title(self):
977  """
978  Determines the type based on the title field.
979  """
980  if 'Title' in self.attributes:
981  title = self.attributes['Title']
982  else:
983  title = self.attributes['title']
984  if title.find('OLCI Level 1b') != -1:
985  self.file_type = 'Level 1B'
986  if 'product_name' in self.attributes:
987  product_name = self.attributes['product_name']
988  if '3A_OL_1_E' in product_name:
989  self.instrument = 'OLCI S3A'
990  elif '3B_OL_1_E' in product_name:
991  self.instrument = 'OLCI S3B'
992  elif (title.find('Level-1') != -1) or (title.find('Level 1') != -1) or \
993  (title.find('L1') != -1):
994  self.file_type, self.instrument = \
995  self._get_data_from_l1_attributes(title)
996  elif title.find('Level-2') != -1 or title.find('Level 2') != -1:
997  self.file_type, self.instrument = \
999  elif (title.find('Level-3') != -1) or (title.find('level-3') != -1):
1000  self.file_type, self.instrument = \
1002  elif title.find('Level-0') != -1:
1003  self.file_type, self.instrument = \
1005  elif title.find('Ancillary') != -1:
1006  self.file_type, self.instrument = \
1008  elif title.find('VIIRS') != -1:
1009  self.instrument = 'VIIRS'
1010  if 'processing_level' in self.attributes:
1011  if self.attributes['processing_level'].upper().find('L1A') != -1:
1012  self.file_type = 'Level 1A'
1013  elif self.attributes['processing_level'].upper().find('L1B') != -1:
1014  self.file_type = 'Level 1B'
1015  elif self.attributes['processing_level'].upper().find('GEO') != -1:
1016  self.file_type = 'GEO'
1017  elif title.find('L1A') != -1:
1018  self.file_type = 'Level 1A'
1019  elif title.find('L1B') != -1:
1020  self.file_type = 'Level 1B'
1021 
1022  def _read_metadata(self):
1023  """
1024  Using the MetaUtils.readMetadata function, reads the metadata and loads
1025  it into the attributes member variable.
1026  """
1027  try:
1028  if os.path.islink(self.file_path):
1029  self.file_path = os.path.realpath(self.file_path)
1031  if not (attrs is None):
1032  self.attributes = attrs
1033  except SystemExit:
1034  # readMetadata calls sys.exit() when it can't find attributes
1035  # to process, but we don't want to exit here.
1036  pass
1037  except TypeError:
1038  pass
def get_timestamp_from_year_day_mil(year, doy, millisecs)
def _create_timestamp_using_YMDTHMS(self, time_str)
def _create_modis_l1_timestamp(self, rng_date, rng_time)
def readMetadata(filename)
Definition: MetaUtils.py:201
def convert_millisecs_to_time_str(millisecs)
def convert_month_day_to_doy(mon, dom, yr)
Definition: time_utils.py:8
def _create_timestamp_using_mon_abbr(self, time_str)
def get_timestamp_from_month_day(sdate, stime)
def _get_time_from_coverage_field(self, coverage_time)
const char * str
Definition: l1c_msi.cpp:35
def _get_landsat_end_time(self, start_yr, start_doy, start_hr, start_min, start_sec)