OB.DAAC Logo
NASA Logo
Ocean Color Science Software

ocssw V2022
anc_utils.py
Go to the documentation of this file.
1 
2 import os
3 import sys
4 import gc
5 import re
6 import subprocess
7 import json
8 from datetime import datetime
9 from datetime import timedelta
10 
11 import xml.etree.ElementTree as ElementTree
12 from operator import sub
13 from collections import OrderedDict
14 
15 import seadasutils.MetaUtils as MetaUtils
16 import seadasutils.ProcUtils as ProcUtils
17 from seadasutils.timestamp_utils import goci_timestamp
18 from seadasutils.timestamp_utils import hawkeye_timestamp
19 from seadasutils.timestamp_utils import hico_timestamp
20 from seadasutils.timestamp_utils import meris_timestamp
21 from seadasutils.timestamp_utils import msi_timestamp
22 from seadasutils.timestamp_utils import ocm2_timestamp
23 from seadasutils.timestamp_utils import olci_timestamp
24 from seadasutils.timestamp_utils import etm_timestamp
25 from seadasutils.timestamp_utils import sgli_timestamp
26 from seadasutils.timestamp_utils import tm_timestamp
27 from seadasutils.timestamp_utils import l9_timestamp
28 from modis.modis_utils import modis_timestamp
29 from viirs.viirs_utils import viirs_timestamp
30 from seadasutils.aquarius_utils import aquarius_timestamp
31 
32 
33 import seadasutils.ancDB as db
34 #import modules.ancDBmysql as db
35 
36 
37 DEFAULT_ANC_DIR_TEXT = "$OCVARROOT"
38 
39 
40 class getanc:
41  """
42  utilities for ancillary file search
43  """
44 
45  def __init__(self, filename=None,
46  start=None,
47  stop=None,
48  ancdir=None,
49  ancdb='ancillary_data.db',
50  anc_file=None,
51  curdir=False,
52  atteph=False,
53  sensor=None,
54  opt_flag=None,
55  verbose=0,
56  printlist=True,
57  download=True,
58  timeout=10,
59  refreshDB=False,
60  use_filename=False):
61  self.filename = filename
62  self.start = start
63  self.stop = stop
64  self.ancdir = ancdir
65  self.ancdb = ancdb
66  self.anc_file=anc_file
67  self.curdir = curdir
68  self.opt_flag = opt_flag
69  self.atteph = atteph
70  self.dl = download
71  self.refreshDB = refreshDB
72  self.sensor = sensor
73  self.dirs = {}
74  self.files = {}
75  self.printlist = printlist
76  self.verbose = verbose
77  self.timeout = timeout
78  self.server_status = None
79  self.db_status = None
80  self.proctype = None
81  self.use_filename=use_filename
82  if self.atteph:
83  self.proctype = 'modisGEO'
84 
85  self.query_site = "oceandata.sci.gsfc.nasa.gov"
86  self.data_site = "oceandata.sci.gsfc.nasa.gov"
87 
88  def chk(self):
89  """
90  Check validity of inputs to
91  """
92 
93  if self.start is None and self.filename is None:
94  print("ERROR: No L1A_or_L1B_file or start time specified!")
95  sys.exit(32)
96 
97  if self.atteph:
98  if self.sensor is None and self.filename is None:
99  print("ERROR: No FILE or MISSION specified.")
100  sys.exit(1)
101  if self.sensor is not None and self.sensor != "modisa" and self.sensor != "modist" \
102  and self.sensor.lower() != "aqua" and self.sensor.lower() != "terra":
103  print("ERROR: Mission must be 'aqua', 'modisa', 'terra', or 'modist' ")
104  sys.exit(16)
105 
106  if self.curdir is True and self.ancdir is not None:
107  print("ERROR: The '--use-current' and '--ancdir' arguments cannot be used together.")
108  print(" Please use only one of these options.")
109  sys.exit(1)
110 
111  if self.start is not None:
112  if (len(self.start) != 13 and len(self.start) != 19) or int(self.start[0:4]) < 1978 or int(self.start[0:4]) > 2030:
113  print("ERROR: Start time must be in YYYYDDDHHMMSS or YYYY-MM-DDTHH:MM:SS format and YYYY is between 1978 and 2030.")
114  sys.exit(1)
115 
116  if self.stop is not None:
117  if (len(self.stop) != 13 and len(self.start) != 19) or int(self.stop[0:4]) < 1978 or int(self.stop[0:4]) > 2030:
118  print("ERROR: End time must be in YYYYDDDHHMMSS or YYYY-MM-DDTHH:MM:SS format and YYYY is between 1978 and 2030.")
119  sys.exit(1)
120 
121  @staticmethod
123  """
124  Extracts and returns the start time, start date, end time, and end date
125  from info. Returns a None value for any item not found.
126  """
127  starttime = None
128  stoptime = None
129  startdate = None
130  stopdate = None
131  for line in info[0].decode("utf-8").splitlines():
132  if line.find("Start_Time") != -1:
133  starttime = line.split('=')[1]
134  if line.find("End_Time") != -1:
135  stoptime = line.split('=')[1]
136  if line.find("Start_Date") != -1:
137  startdate = line.split('=')[1]
138  if line.find("End_Date") != -1:
139  stopdate = line.split('=')[1]
140  return starttime, startdate, stoptime, stopdate
141 
142  @staticmethod
143  def get_goci_time(goci_time_str):
144  month_dict = {'JAN': '01', 'FEB': '02', 'MAR': '03', 'APR': '04',
145  'MAY': '05', 'JUN': '06', 'JUL': '07', 'AUG': '08',
146  'SEP': '09', 'OCT': '10', 'NOV': '11', 'DEC': '12'}
147  time_str = goci_time_str[8:12] + '-' + \
148  month_dict[goci_time_str[4:7]] + '-' + \
149  goci_time_str[1:3] + 'T' + goci_time_str[13:15] + ':' + \
150  goci_time_str[16:18] + ':' + goci_time_str[19:21] + '.' + \
151  goci_time_str[22:25] + 'Z'
152  return time_str
153 
154  @staticmethod
156  data_elem = elem.find('Data')
157  value_elem = data_elem.find('DataFromFile')
158  return value_elem.text.strip()
159 
160  def get_start_end_info_from_xml(self, raw_xml):
161  """
162  Extracts and returns the start time, start date, end time, and end date
163  from the XML tree in raw_xml. Returns a None value for any item not
164  found.
165  """
166 
167  xml_root = ElementTree.fromstring(raw_xml)
168 
169  time_start_list = xml_root.findall('.//Attribute[@Name="time_coverage_start"]')
170  if len(time_start_list) > 0:
171  if len(time_start_list) > 1:
172  print("Encountered more than 1 time_coverage_start tag. Using 1st value.")
173  start = self.get_time_coverage_xml(time_start_list[0])
174  else:
175  time_start_list = xml_root.findall('.//Attribute[@Name="Scene Start time"]')
176  if len(time_start_list) > 1:
177  print("Encountered more than 1 Scene Start time tag. Using 1st value.")
178  start_str = self.get_time_coverage_xml(time_start_list[0])
179  start = self.get_goci_time(start_str)
180 
181  time_end_list = xml_root.findall('.//Attribute[@Name="time_coverage_end"]')
182  if len(time_end_list) > 0:
183  if len(time_end_list) > 1:
184  print("Encountered more than 1 time_coverage_end tag. Using 1st value.")
185  stop = self.get_time_coverage_xml(time_end_list[0])
186  else:
187  time_end_list = xml_root.findall('.//Attribute[@Name="Scene end time"]')
188  if len(time_end_list) > 1:
189  print("Encountered more than 1 Scene end time tag. Using 1st value.")
190  stop_str = self.get_time_coverage_xml(time_end_list[0])
191  stop = self.get_goci_time(stop_str)
192  return start, stop
193 
194  def setup(self):
195  """
196  Set up the basics
197  """
198  # global stopdate, stoptime, startdate, starttime
199 
200  # set l2gen parameter filename
201  if self.filename is None:
202  if len(self.start) == 13:
203  start_obj = datetime.strptime(self.start, '%Y%j%H%M%S')
204  self.start = datetime.strftime(start_obj, '%Y-%m-%dT%H:%M:%S')
205  if self.stop is not None:
206  stop_obj = datetime.strptime(self.stop, '%Y%j%H%M%S')
207  self.stop = datetime.strftime(stop_obj, '%Y-%m-%dT%H:%M:%S')
208  if self.anc_file is None:
209  self.base = self.start
210  else:
211  self.base = self.anc_file.replace(".anc", "")
212  self.server_file = self.base + ".anc.server"
213 
214  if self.atteph:
215  self.anc_file = self.base + ".atteph"
216  else:
217  self.anc_file = self.base + ".anc"
218  elif self.use_filename:
219  if self.anc_file is None:
220  self.base = os.path.basename(self.filename)
221  self.server_file = "{0:>s}.anc.server".format('.'.join(self.base.split('.')[0:-1]))
222  if self.atteph:
223  self.anc_file = "{0:>s}.atteph".format('.'.join(self.base.split('.')[0:-1]))
224  else:
225  self.anc_file = "{0:>s}.anc".format('.'.join(self.base.split('.')[0:-1]))
226  else:
227  self.base = self.anc_file.replace(".anc", "")
228  self.server_file = self.base + ".anc.server"
229 
230  if self.atteph:
231  self.anc_file = '.'.join([self.base, 'atteph'])
232  else:
233  self.anc_file = '.'.join([self.base, 'anc'])
234 
235  if self.server_file == '.anc.server':
236  self.server_file = self.filename + self.server_file
237  self.anc_file = self.filename + self.anc_file
238  else:
239  if self.anc_file is None:
240  self.base = os.path.basename(self.filename)
241  self.server_file = "{0:>s}.anc.server".format('.'.join(self.base.split('.')[0:-1]))
242  else:
243  self.base = self.anc_file.replace(".anc", "")
244  self.server_file = self.base + ".anc.server"
245 
246  if self.atteph:
247  self.anc_file = '.'.join([self.base, 'atteph'])
248  else:
249  self.anc_file = '.'.join([self.base, 'anc'])
250 
251  if self.server_file == '.anc.server':
252  self.server_file = self.filename + self.server_file
253  self.anc_file = self.filename + self.anc_file
254 
255  # Check if start time specified.. if not, obtain from HDF header or if the
256  # file doesn't exist, obtain start time from filename
257  if self.start is None:
258  # Check for existence of ifile, and if it doesn't exist assume the
259  # user wants to use this script without an actual input file, but
260  # instead an OBPG formatted filename that indicates the start time.
261  if not os.path.exists(self.filename):
262  if self.sensor:
263  print("*** WARNING: Input file doesn't exist! Parsing filename for start time and setting")
264  print("*** end time to 5 minutes later for MODIS and 15 minutes later for other sensors.")
265  if len(self.base) < 14 or int(self.base[1:5]) < 1978 or int(self.base[1:5]) > 2030:
266  print("ERROR: Filename must be in XYYYYDDDHHMMSS format where X is the")
267  print("sensor letter and YYYY is between 1978 and 2030.")
268  sys.exit(1)
269  else:
270  self.start = self.base[1:14]
271  else:
272  print("*** ERROR: Input file doesn't exist and mission not set...bailing out...")
273  sys.exit(1)
274 
275  else:
276  # Determine start/end times from HDF file
277  # for l1info subsample every 250 lines
278  if self.verbose:
279  print("Determining pass start and end times...")
280  senchk = ProcUtils.check_sensor(self.filename)
281 
282  if re.search('(Aqua|Terra)', senchk):
283  # if self.mission == "A" or self.mission == "T":
284  self.start, self.stop, self.sensor = modis_timestamp(self.filename)
285  elif senchk.find("viirs") == 0:
286  self.start, self.stop, self.sensor = viirs_timestamp(self.filename)
287  elif senchk.find("aquarius") == 0:
288  self.start, self.stop, self.sensor = aquarius_timestamp(self.filename)
289  elif re.search('goci', senchk):
290  self.start, self.stop, self.sensor = goci_timestamp(self.filename)
291  elif re.search('hawkeye', senchk):
292  self.start, self.stop, self.sensor = hawkeye_timestamp(self.filename)
293  elif senchk.find("hico") == 0:
294  self.start, self.stop, self.sensor = hico_timestamp(self.filename)
295  elif re.search('meris', senchk):
296  self.start, self.stop, self.sensor = meris_timestamp(self.filename)
297  elif re.search('(S2A|S2B)', senchk):
298  self.start, self.stop, self.sensor = msi_timestamp(self.filename)
299  elif re.search('ocm2', senchk):
300  self.start, self.stop, self.sensor = ocm2_timestamp(self.filename)
301  elif re.search('ETM', senchk):
302  self.start, self.stop, self.sensor = etm_timestamp(self.filename)
303  elif re.search('(3A|3B)', senchk):
304  self.start, self.stop, self.sensor = olci_timestamp(self.filename)
305  elif re.search('sgli', senchk):
306  self.start, self.stop, self.sensor = sgli_timestamp(self.filename)
307  elif re.search('tm', senchk):
308  self.start, self.stop, self.sensor = tm_timestamp(self.filename)
309  elif re.search('L9', senchk):
310  self.start, self.stop, self.sensor = l9_timestamp(self.filename)
311  else:
312  if self.sensor is None:
313  self.sensor = senchk
314  mime_data = MetaUtils.get_mime_data(self.filename)
315  if MetaUtils.is_netcdf4(mime_data):
316  metadata = MetaUtils.dump_metadata(self.filename)
317  starttime, stoptime = self.get_start_end_info_from_xml(metadata)
318  starttime = starttime.strip('"')
319  stoptime = stoptime.strip('"')
320  starttime = starttime.strip("'")
321  stoptime = stoptime.strip("'")
322  if starttime.find('T') != -1:
323  self.start = starttime[0:19]
324  # self.start = ProcUtils.date_convert(starttime, 't', 'j')
325  else:
326  self.start = starttime[0:19]
327  # self.start = ProcUtils.date_convert(starttime, 'h', 'j')
328  if stoptime.find('T') != -1:
329  self.stop = stoptime[0:19]
330  # self.stop = ProcUtils.date_convert(stoptime, 't', 'j')
331  else:
332  self.stop = stoptime[0:19]
333  # self.stop = ProcUtils.date_convert(stoptime, 'h', 'j')
334  pass
335  else:
336  infocmd = [os.path.join(self.dirs['bin'], 'l1info'), '-s', '-i 250', self.filename]
337  l1info = subprocess.Popen(infocmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
338  info = l1info.communicate()
339  (starttime, startdate, stoptime, stopdate) = self.get_start_end_info(info)
340  if not starttime or not startdate or not stoptime or not stopdate:
341  err_msg = 'ERROR: For ' + self.base + ' could not determine: '
342  if not starttime:
343  err_msg = err_msg + ' start time'
344  if not startdate:
345  if not starttime:
346  err_msg = err_msg + ', start date'
347  else:
348  err_msg = err_msg + ' start date'
349  if not stoptime:
350  if not starttime or not startdate:
351  err_msg = err_msg + ', stoptime'
352  else:
353  err_msg = err_msg + ' stop time'
354  if not stopdate:
355  if not starttime or not startdate or not stoptime:
356  err_msg = err_msg + ', stop date'
357  else:
358  err_msg = err_msg + ' stop date'
359  err_msg = err_msg + '. Exiting.'
360  print(err_msg)
361  if info[1]:
362  print("l1info reported the following error:")
363  print(' {0}'.format(info[1]))
364  sys.exit(1)
365  else:
366  # self.start = ProcUtils.date_convert(startdate + ' ' + starttime, 'h', 'j')
367  # self.stop = ProcUtils.date_convert(stopdate + ' ' + stoptime, 'h', 'j')
368  self.start = startdate + 'T' + starttime[0:8]
369  self.stop = stopdate + 'T' + stoptime[0:8]
370 
371  if self.filename and not self.sensor and not self.use_filename:
372  # Make sure sensor is set (JIRA Ticket #1012)
373  self.sensor = ProcUtils.check_sensor(self.filename)
374 
375  if self.verbose:
376  print()
377  print("Input file: " + str(self.filename))
378  print("Sensor : " + str(self.sensor))
379  print("Start time: " + str(self.start))
380  print("End time : " + str(self.stop))
381  print()
382 
383  def set_opt_flag(self, key, off=False):
384  """
385  set up the opt_flag for display_ancillary_data (type=anc)
386  opt_flag values:
387  0 - just the basics, MET/OZONE
388  1 - include OISST
389  2 - include NO2
390  4 - include ICE
391  """
392  optkey = {'sst': 1, 'no2': 2, 'ice': 4}
393 
394  if off:
395  self.opt_flag = self.opt_flag - optkey[key]
396  else:
397  self.opt_flag = self.opt_flag + optkey[key]
398 
399  def finddb(self):
400  """
401  Checks local db for anc files.
402  """
403  print(self.ancdb)
404  if len(os.path.dirname(self.ancdb)):
405  self.dirs['log'] = os.path.dirname(self.ancdb)
406  self.ancdb = os.path.basename(self.ancdb)
407 
408  if not os.path.exists(self.dirs['log']):
409  self.ancdb = os.path.basename(self.ancdb)
410  print('''Directory %s does not exist.
411 Using current working directory for storing the ancillary database file: %s''' % (self.dirs['log'], self.ancdb))
412  self.dirs['log'] = self.dirs['run']
413 
414  self.ancdb = os.path.join(self.dirs['log'], self.ancdb)
415 
416  if not os.path.exists(self.ancdb):
417  return 0
418 
419  ancdatabase = db.ancDB(dbfile=self.ancdb)
420  if not os.path.getsize(self.ancdb):
421  if self.verbose:
422  print("Creating database: %s " % self.ancdb)
423  ancdatabase.openDB()
424  ancdatabase.create_db()
425  else:
426  ancdatabase.openDB()
427  if self.verbose:
428  print("Searching database: %s " % self.ancdb)
429 
430  if self.filename:
431  filekey = os.path.basename(self.filename)
432  else:
433  filekey = None
434  if self.start and len(self.start) == 13:
435  start_obj = datetime.strptime(self.start, '%Y%j%H%M%S')
436  self.start = datetime.strftime(start_obj, '%Y-%m-%dT%H:%M:%S')
437  if self.stop is not None:
438  stop_obj = datetime.strptime(self.stop, '%Y%j%H%M%S')
439  self.stop = datetime.strftime(stop_obj, '%Y-%m-%dT%H:%M:%S')
440  status = ancdatabase.check_file(filekey,starttime=self.start)
441  if status:
442  if not self.refreshDB:
443  self.files = ancdatabase.get_ancfiles(filekey, self.atteph, starttime=self.start)
444  if self.curdir:
445  for anckey in list(self.files.keys()):
446  self.files[anckey] = os.path.basename(self.files[anckey])
447  self.db_status = ancdatabase.get_status(filekey, self.atteph, starttime=self.start)
448  self.start, self.stop = ancdatabase.get_filetime(filekey, starttime=self.start)
449  else:
450  ancdatabase.delete_record(filekey, starttime=self.start)
451 
452  ancdatabase.closeDB()
453 
454  if status and self.db_status:
455  if not self.refreshDB:
456  if self.db_status > 0:
457  print("Warning! Non-optimal data exist in local repository.")
458  print("Consider re-running with the --refreshDB option to check for optimal ancillary files")
459  return 1
460  else:
461  return 0
462  else:
463  return 0
464 
465  def findweb(self):
466  """
467  Execute the display_ancillary_files search and populate the locate cache database
468  """
469 
470  # dlstat = 0
471 
472  with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'missionID.json'), 'r') as msn_file:
473  msn = json.load(msn_file)
474 
475  # msn = {"seawifs": "6", "modisa": "7", "aqua": "7", "modist": "8", "terra": "8", "octs": "9",
476  # "czcs": "11", "suomi-npp": "14", "aquarius": "15", "meris": "19", "ocm2": "21", "hico" : "25",
477  # "goci": "27", "oli": "28", "3a": "29", "jpss-1": "33","hawkeye": "35", "3b": "36"}
478 
479  ProcUtils.remove(self.server_file)
480 
481  # Query the OBPG server for the ancillary file list
482  opt_flag = str(self.opt_flag)
483  anctype = 'anc_data_api'
484  # if self.atteph:
485  # opt_flag = ''
486  # anctype = 'atteph_test'
487 
488  if self.sensor == 'aquarius':
489  opt_flag = ''
490 
491  msnchar = '0'
492  if str(self.sensor).lower() in msn:
493  # msnchar = msn[str(self.sensor).lower()]
494  msnchar = msn[str(self.sensor.lower())]
495  elif self.sensor and self.sensor.isdigit():
496  msnchar = self.sensor
497 
498  if self.stop is None and not self.use_filename:
499  start_obj = datetime.strptime(self.start, '%Y-%m-%dT%H:%M:%S')
500  time_change = timedelta(minutes=5)
501  stop_obj = start_obj + time_change
502  self.stop = datetime.strftime(stop_obj,'%Y-%m-%dT%H:%M:%S')
503  anc_str = '?&m=' + msnchar + '&s=' + self.start + '&e=' + self.stop + '&missing_tags=1'
504  dlstat = ProcUtils.httpdl(self.query_site, '/'.join(['/api', anctype, anc_str]),
505  os.path.abspath(os.path.dirname(self.server_file)),
506  outputfilename=self.server_file,
507  timeout=self.timeout,
508  verbose=self.verbose
509  )
510  elif self.use_filename:
511  anc_str = '?filename=' + os.path.basename(self.filename) + '&missing_tags=1'
512  dlstat = ProcUtils.httpdl(self.query_site,
513  '/'.join(['/api', anctype, anc_str]),
514  os.path.abspath(os.path.dirname(self.server_file)),
515  outputfilename=self.server_file,
516  timeout=self.timeout,
517  verbose=self.verbose
518  )
519  else:
520  anc_str = '?&m=' + msnchar + '&s=' + self.start + '&e=' + self.stop + '&missing_tags=1'
521  dlstat = ProcUtils.httpdl(self.query_site,
522  '/'.join(['/api', anctype, anc_str]),
523  os.path.abspath(os.path.dirname(self.server_file)),
524  outputfilename=self.server_file,
525  timeout=self.timeout,
526  verbose=self.verbose
527  )
528  gc.collect()
529 
530  if dlstat:
531  print("Error retrieving ancillary file list")
532  sys.exit(dlstat)
533 
534  with open(self.server_file, 'r') as data_file:
535  results = json.load(data_file)
536  self.db_status = int(results['status'])
537 
538  if self.db_status == 128:
539  ProcUtils.remove(self.anc_file)
540  print("The time duration provided is longer than 2 hours. please try with a shorter duration." )
541  print("No parameter file created.")
542  ProcUtils.remove(self.server_file)
543  sys.exit(128)
544 
545  if self.db_status == 64:
546  ProcUtils.remove(self.anc_file)
547  print("Stop time is earlier than start time." )
548  print("No parameter file created.")
549  ProcUtils.remove(self.server_file)
550  sys.exit(64)
551 
552  self.db_status = 0
553 
554  for f in results['files']:
555  if (len(str(f[1]))) == 0:
556  if not self.atteph:
557  if re.search('MET', f[0]):
558  self.db_status = self.db_status | 1
559  if re.search('OZONE', f[0]):
560  self.db_status = self.db_status | 2
561  if re.search('sst', f[0]):
562  self.db_status = self.db_status | 4
563  if re.search('no2', f[0]):
564  self.db_status = self.db_status | 8
565  if re.search('ice', f[0]):
566  self.db_status = self.db_status | 16
567  if self.atteph:
568  if re.search('ATT', f[0]):
569  self.db_status = self.db_status | 4
570  if re.search('EPH', f[0]):
571  self.db_status = self.db_status | 8
572  else:
573  if self.atteph:
574  if re.search('ATT|EPH', f[0]):
575  self.files[str(f[0]).lower()] = str(f[1])
576  if (f[2] != 1):
577  if re.search('ATT', f[0]):
578  self.db_status = self.db_status | 1
579  if re.search('EPH', f[0]):
580  self.db_status = self.db_status | 2
581  else:
582  if re.search('MET|OZONE|sst|ice|AER|GEO', f[0]):
583  self.files[str(f[0]).lower()] = str(f[1])
584  # self.db_status = int(results['status'])
585 
586  # FOR MET/OZONE:
587  # For each anc type, DB returns either a zero status if all optimal files are
588  # found, or different error statuses if not. However, 3 MET or OZONE files can be
589  # returned along with an error status meaning there were one or more missing
590  # files that were then filled with the file(s) found, and so though perhaps
591  # better than climatology it's still not optimal. Therefore check for cases
592  # where all 3 MET/ozone files are returned but status is negative and then
593  # warn the user there might be more files to come and they should consider
594  # reprocessing at a later date.
595  #
596  # DB return status bitwise values:
597  # -all bits off means all is well in the world
598  # -bit 0 = 1 - missing one or more MET
599  # -bit 1 = 1 - missing one or more OZONE
600  # -bit 2 = 1 - missing SST
601  # -bit 3 = 1 - missing NO2
602  # -bit 4 = 1 - missing ICE
603 
604  # FOR ATT/EPH:
605  # all bits off means all is well in the world
606  # -bit 0 = 1 - predicted attitude selected
607  # -bit 1 = 1 - predicted ephemeris selected
608  # -bit 2 = 1 - no attitude found
609  # -bit 3 = 1 - no ephemeris found
610 
611  if self.server_status == 1 or dlstat or self.db_status is None:
612  print("ERROR: The display_ancillary_files.cgi script encountered an error and returned the following text:")
613  print()
614  ProcUtils.cat(self.server_file)
615  sys.exit(99)
616 
617  if self.db_status == 31 and not self.atteph:
618  ProcUtils.remove(self.anc_file)
619  print("No ancillary files currently exist that correspond to the start time " + self.start)
620  print("No parameter file created (l2gen defaults to the climatologies).")
621  ProcUtils.remove(self.server_file)
622  sys.exit(31)
623 
624  if self.db_status == 12 and self.atteph:
625  ProcUtils.remove(self.anc_file)
626  print("No att/eph files currently exist that correspond to the start time " + self.start)
627  ProcUtils.remove(self.server_file)
628  sys.exit(12)
629 
630  # extra checks
631  for f in (list(self.files.keys())):
632  if not len(self.files[f]):
633  print("ERROR: display_ancillary_files.cgi script returned blank entry for %s. Exiting." % f)
634  sys.exit(99)
635 
636  ancdatabase = db.ancDB(dbfile=self.ancdb)
637 
638  if not os.path.exists(ancdatabase.dbfile) or os.path.getsize(ancdatabase.dbfile) == 0:
639  ancdatabase.openDB()
640  ancdatabase.create_db()
641  else:
642  ancdatabase.openDB()
643 
644  missing = []
645 
646  for anctype in self.files:
647  if self.files[anctype] == 'missing':
648  missing.append(anctype)
649  continue
650  if (self.filename and self.dl) or (self.start and self.dl):
651  path = self.dirs['anc']
652  if not self.curdir:
653  year, day = self.yearday(self.files[anctype])
654  path = os.path.join(path, year, day)
655 
656  if self.filename:
657  filekey = os.path.basename(self.filename)
658  else:
659  filekey = None
660  ancdatabase.insert_record(satfile=filekey, starttime=self.start, stoptime=self.stop, anctype=anctype,
661  ancfile=self.files[anctype], ancpath=path, dbstat=self.db_status,
662  atteph=self.atteph)
663 
664  ancdatabase.closeDB()
665  # remove missing items
666  for anctype in missing:
667  self.files.__delitem__(anctype)
668 
669  def yearday(self, ancfile):
670  if ancfile.startswith('RIM_'):
671  ymd = ancfile.split('_')[2]
672  dt = datetime.strptime(ymd, '%Y%m%d')
673  year = dt.strftime('%Y')
674  day = dt.strftime('%j')
675  return year, day
676 
677  if ancfile.startswith('MERRA'):
678  ymd = ancfile.split('.')[4]
679  dt = datetime.strptime(ymd, '%Y%m%d')
680  year = dt.strftime('%Y')
681  day = dt.strftime('%j')
682  return year, day
683 
684  # For YYYYmmddT or YYmmddHHMMSS
685  if re.search('[\d]{8}T', ancfile) or re.search('[\d]{14}', ancfile):
686  ymd = re.search('[\d]{8}', ancfile).group()
687  dt = datetime.strptime(ymd, '%Y%m%d')
688  year = dt.strftime('%Y')
689  day = dt.strftime('%j')
690  return year, day
691  # For YYYYDDDHHMMSS
692  elif re.search('[\d]{13}', ancfile):
693  ymd = re.search('[\d]{7}', ancfile).group()
694  year = ymd[0:4]
695  day = ymd[4:7]
696  return year, day
697  # For YYYYmmddHHMM
698  elif re.search('[\d]{12}', ancfile):
699  ymd = re.search('[\d]{8}', ancfile).group()
700  dt = datetime.strptime(ymd, '%Y%m%d')
701  year = dt.strftime('%Y')
702  day = dt.strftime('%j')
703  return year, day
704  # For YYYYDDDHHMM
705  elif re.search('[\d]{11}', ancfile):
706  ymd = re.search('[\d]{7}', ancfile).group()
707  year = ymd[0:4]
708  day = ymd[4:7]
709  return year, day
710  # For YYYYmmddHH
711  elif re.search('[\d]{10}', ancfile):
712  ymd = re.search('[\d]{8}', ancfile).group()
713  dt = datetime.strptime(ymd, '%Y%m%d')
714  year = dt.strftime('%Y')
715  day = dt.strftime('%j')
716  return year, day
717  # For YYYYDDDHH
718  elif re.search('[\d]{9}', ancfile):
719  ymd = re.search('[\d]{7}', ancfile).group()
720  year = ymd[0:4]
721  day = ymd[4:7]
722  return year, day
723  # For YYYYmmdd
724  elif re.search('[\d]{8}', ancfile):
725  ymd = re.search('[\d]{8}', ancfile).group()
726  dt = datetime.strptime(ymd, '%Y%m%d')
727  year = dt.strftime('%Y')
728  day = dt.strftime('%j')
729  return year, day
730  # For YYYYDDD
731  elif re.search('[\d]{7}', ancfile):
732  ymd = re.search('[\d]{7}', ancfile).group()
733  year = ymd[0:4]
734  day = ymd[4:7]
735  return year, day
736 
737  if ancfile.startswith('GMAO'):
738  ymd = ancfile.split('.')[1][0:8]
739  dt = datetime.strptime(ymd, '%Y%m%d')
740  year = dt.strftime('%Y')
741  day = dt.strftime('%j')
742  return year, day
743 
744  if ancfile[:1].isdigit():
745  ymd = ancfile[0:8]
746  dt = datetime.strptime(ymd, '%Y%m%d')
747  year = dt.strftime('%Y')
748  day = dt.strftime('%j')
749  return year, day
750 
751  if ancfile.startswith('SIF'):
752  yyyyddd = ancfile.split('.')[0]
753  offset = 3
754  elif ancfile.startswith('PERT_'):
755  yyyyddd = ancfile.split('_')[2]
756  offset = 0
757  elif self.atteph and not re.search(".(att|eph)$", ancfile):
758  yyyyddd = ancfile.split('.')[1]
759  offset = 1
760  else:
761  yyyyddd = ancfile
762  offset = 1 # skip only 1st char
763 
764  year = yyyyddd[offset:offset + 4]
765  day = yyyyddd[offset + 4:offset + 7]
766  return year, day
767 
768  def locate(self, forcedl=False):
769  """
770  Find the files on the local system or download from OBPG
771  """
772 
773  FILES = []
774  for f in (list(self.files.keys())):
775  if self.atteph:
776  if re.search('scat|atm|met|ozone|file|geo|aer', f):
777  continue
778  else:
779  if re.search('att|eph', f):
780  continue
781  FILES.append(os.path.basename(self.files[f]))
782 
783  dl_msg = 1
784 
785  for FILE in list(OrderedDict.fromkeys(FILES)):
786  year, day = self.yearday(FILE)
787 
788  # First check hard disk...unless forcedl is set
789 
790  if self.curdir:
791  self.dirs['path'] = '.' # self.dirs['anc']
792  if os.path.exists(FILE) and forcedl is False:
793  download = 0
794  if self.verbose:
795  print(" Found: %s" % FILE)
796  else:
797  download = 1
798  else:
799  ancdir = self.dirs['anc']
800 
801  self.dirs['path'] = os.path.join(ancdir, year, day)
802  if os.path.exists(os.path.join(ancdir, year, day, FILE)) and forcedl is False:
803  download = 0
804  if self.verbose:
805  print(" Found: %s/%s" % (self.dirs['path'], FILE))
806  else:
807  download = 1
808 
809  # Not on hard disk, download the file
810 
811  if download == 1:
812  if self.dl is False:
813  if dl_msg == 1:
814  if self.verbose:
815  print("Downloads disabled. The following missing file(s) will not be downloaded:")
816  dl_msg = 0
817  if self.verbose:
818  print(" " + FILE)
819  else:
820 
821  if self.verbose:
822  print("Downloading '" + FILE + "' to " + self.dirs['path'])
823  status = ProcUtils.httpdl(self.data_site, ''.join(['/ob/getfile/', FILE]),
824  self.dirs['path'], timeout=self.timeout, uncompress=True,force_download=forcedl,
825  verbose=self.verbose)
826  gc.collect()
827  if status:
828  if status == 304:
829  if self.verbose:
830  print("%s is not newer than local copy, skipping download" % FILE)
831  elif status == 401:
832  print("*** ERROR: Authentication Failure retrieving:")
833  print("*** " + '/'.join([self.data_site, 'ob/getfile', FILE]))
834  print("*** Please check that your ~/.netrc file is setup correctly and has proper permissions.")
835  print("***")
836  print("*** see: https://oceancolor.gsfc.nasa.gov/data/download_methods/")
837  print("***\n")
838  else:
839  print("*** ERROR: The HTTP transfer failed with status code " + str(status) + ".")
840  print("*** Please check your network connection and for the existence of the remote file:")
841  print("*** " + '/'.join([self.data_site, 'ob/getfile', FILE]))
842  print("***")
843  print("*** Also check to make sure you have write permissions under the directory:")
844  print("*** " + self.dirs['path'])
845  print()
846 
847  if status != 304:
848  ProcUtils.remove(os.path.join(self.dirs['path'], FILE))
849  ProcUtils.remove(self.server_file)
850  sys.exit(1)
851 
852  for f in (list(self.files.keys())):
853  if self.atteph:
854  if re.search('met|ozone|file|aer|geo', f):
855  continue
856  else:
857  if re.search('att|eph', f):
858  continue
859  if FILE == self.files[f]:
860  self.files[f] = os.path.join(self.dirs['path'], FILE)
861 
862  def write_anc_par(self):
863  """
864  create the .anc parameter file
865  """
866  ProcUtils.remove(self.anc_file)
867 
868  NONOPT = ""
869  if not self.atteph:
870 
871  if self.sensor == 'aquarius':
872  inputs = {'MET': {'bitval': 1, 'required': ['met1', 'met2', 'atm1', 'atm2']},
873  'SST': {'bitval': 4, 'required': ['sstfile1', 'sstfile2']},
874  'SeaIce': {'bitval': 16, 'required': ['icefile1', 'icefile2']},
875  'Salinity': {'bitval': 32, 'required': ['sssfile1', 'sssfile2']},
876  'XRAY': {'bitval': 64, 'required': ['xrayfile1']},
877  # 'SCAT': {'bitval': 128, 'required': ['scat']},
878  'TEC': {'bitval': 256, 'required': ['tecfile']},
879  'SWH': {'bitval': 512, 'required': ['swhfile1', 'swhfile2']},
880  'Frozen': {'bitval': 1024, 'required': ['frozenfile1', 'frozenfile2']},
881  'GEOS': {'bitval': 2048, 'required': ['geosfile']},
882  'ARGOS': {'bitval': 4096, 'required': ['argosfile1', 'argosfile2']},
883  'SIF': {'bitval': 8192, 'required': ['sif']},
884  'PERT': {'bitval': 16384, 'required': ['pert']},
885  'Matchup': {'bitval': 32768, 'required': ['sssmatchup']},
886  'Rainfall': {'bitval': 65536, 'required': ['rim_file']}}
887  for anc in inputs:
888  if self.db_status & inputs[anc]['bitval']:
889  NONOPT = " ".join([NONOPT, anc])
890  else:
891  for ancfile in (inputs[anc]['required']):
892  if ancfile not in self.files:
893  NONOPT = " ".join([NONOPT, anc])
894  print('*** WARNING: No optimal {0} files found.'.format(ancfile))
895  break
896 
897  else: # not aquarius
898 
899  if self.db_status & 1:
900  NONOPT = " ".join([NONOPT, 'MET'])
901  else:
902  for key in (['met1', 'met2', 'met3']):
903  if key not in self.files:
904  NONOPT = " ".join([NONOPT, 'MET'])
905  print("*** WARNING: No optimal MET files found.")
906  break
907 
908  if self.db_status & 2:
909  NONOPT = " ".join([NONOPT, 'OZONE'])
910  else:
911  for key in (['ozone1', 'ozone2', 'ozone3']):
912  if key not in self.files:
913  NONOPT = " ".join([NONOPT, 'OZONE'])
914  print("*** WARNING: No optimal OZONE files found.")
915  break
916 
917  if self.opt_flag & 1 and ('sstfile' not in self.files or (self.db_status & 4)):
918  NONOPT = " ".join([NONOPT, 'SST'])
919  print("*** WARNING: No optimal SST files found.")
920 
921  if self.opt_flag & 2 and ('no2file' not in self.files or (self.db_status & 8)):
922  NONOPT = " ".join([NONOPT, 'NO2'])
923  print("*** WARNING: No optimal NO2 files found.")
924 
925  if self.opt_flag & 4 and ('icefile' not in self.files or (self.db_status & 16)):
926  NONOPT = " ".join([NONOPT, 'Sea Ice'])
927  print("*** WARNING: No optimal ICE files found.")
928 
929  ancpar = open(self.anc_file, 'w')
930 
931  for key in sorted(self.files.keys()):
932  if self.atteph:
933  if key.find('att') != -1 or key.find('eph') != -1:
934  ancpar.write('='.join([key, self.files[key]]) + '\n')
935  else:
936  if key.find('att') == -1 and key.find('eph') == -1:
937  ancpar.write('='.join([key, self.files[key]]) + '\n')
938 
939  ancpar.close()
940 
941  if self.verbose:
942  if self.atteph:
943  if self.dl:
944  print("All required attitude and ephemeris files successfully determined and downloaded.")
945  else:
946  print("All required attitude and ephemeris files successfully determined.")
947  else:
948  print()
949  print("Created '" + self.anc_file + "' l2gen parameter text file:\n")
950 
951  if self.verbose or self.printlist:
952  ProcUtils.cat(self.anc_file)
953 
954  if len(NONOPT):
955  if self.db_status == 31:
956  print("No optimal ancillary files were found.")
957  print("No parameter file was created (l2gen defaults to the climatological ancillary data).")
958  print("Exiting.")
959  ProcUtils.remove(self.server_file)
960  else:
961  print()
962  print("*** WARNING: The following ancillary data types were missing or are not optimal: " + NONOPT)
963  if self.db_status == 3:
964  print("*** Beware that certain MET and OZONE files just chosen by this program are not optimal.")
965  print("*** For near real-time processing the remaining files may become available soon.")
966  elif self.db_status == 1:
967  print("*** Beware that certain MET files just chosen by this program are not optimal.")
968  print("*** For near real-time processing the remaining files may become available soon.")
969  elif self.db_status == 2:
970  print("*** Beware that certain OZONE files just chosen by this program are not optimal.")
971  print("*** For near real-time processing the remaining files may become available soon.")
972  else:
973  if self.verbose:
974  if self.dl:
975  print()
976  print("- All optimal ancillary data files were determined and downloaded. -")
977  else:
978  print()
979  print("- All optimal ancillary data files were determined. -")
980 
981  def cleanup(self):
982  """
983  remove the temporary 'server' file and adjust return status - if necessary
984  """
985 
986  ProcUtils.remove(self.server_file)
987 
988  # if an anc type was turned off but its db_status bit was on, turn off the
989  # status bit so the user (and GUI) won't think anything's wrong
990  if not self.atteph:
991  if self.db_status & 4 and self.opt_flag & 1:
992  self.db_status = sub(self.db_status, 4)
993  if self.db_status & 8 and self.opt_flag & 2:
994  self.db_status = sub(self.db_status, 8)
995  if self.db_status & 16 and self.opt_flag & 4:
996  self.db_status = sub(self.db_status, 16)
def set_opt_flag(self, key, off=False)
Definition: anc_utils.py:383
def get_start_end_info_from_xml(self, raw_xml)
Definition: anc_utils.py:160
list(APPEND LIBS ${PGSTK_LIBRARIES}) add_executable(atteph_info_modis atteph_info_modis.c) target_link_libraries(atteph_info_modis $
Definition: CMakeLists.txt:7
def get_goci_time(goci_time_str)
Definition: anc_utils.py:143
def __init__(self, filename=None, start=None, stop=None, ancdir=None, ancdb='ancillary_data.db', anc_file=None, curdir=False, atteph=False, sensor=None, opt_flag=None, verbose=0, printlist=True, download=True, timeout=10, refreshDB=False, use_filename=False)
Definition: anc_utils.py:45
def modis_timestamp(arg)
Definition: modis_utils.py:152
#define isdigit(c)
def yearday(self, ancfile)
Definition: anc_utils.py:669
const char * str
Definition: l1c_msi.cpp:35
def viirs_timestamp(arg)
Definition: viirs_utils.py:4
def locate(self, forcedl=False)
Definition: anc_utils.py:768