LutUtils.py

Go to the documentation of this file.

 
import os
import seadasutils.JsonUtils as Session
import seadasutils.SensorUtils as SensorUtils
 
 
def lut_version(lut_name):
    import re
    f = os.path.basename(lut_name)
 
    # lut_name like xcal_<sensor>_<version>_<wavelength>.hdf
    if re.match('xcal', f):
        parts = re.split('([_.])', f)
        return ''.join(parts[4:-4])
 
    # lut_name like <stuff>[_.][vV]<version>.<suffix>
    v = re.search('[_.][vV]\d+', f)
    if v:
        parts = re.split('([_.])', f[v.start() + 1:])
        return ''.join(parts[0:-2])
 
    # no version in lut_name
    return None
 
 
def any_version(lut_name):
    import re
    d = os.path.dirname(lut_name)
    f = os.path.basename(lut_name)
    v = lut_version(f)
    if v:
        p = re.compile(v)
        return os.path.join(d, p.sub('*', f))
    else:
        return lut_name
 
 
def old_version(newfile):
    import glob
    similar = glob.glob(any_version(newfile))
    deletable = [f for f in similar if
                 os.path.getmtime(f) < os.path.getmtime(newfile)]
    return deletable
 
 
def purge_luts(newfiles, verbose=False):
    deletable = [old_version(newf) for newf in newfiles]
    # flatten list of lists into single list
    deletable = [oldf for sublist in deletable for oldf in sublist]
    if len(deletable) > 0:
        if verbose:
            print('\n...deleting outdated LUTs:')
        for f in sorted(deletable):
            if verbose:
                print('- ' + os.path.basename(f))
                os.remove(f)
 
 
class LutUtils:
    """
    Utilities to update various LUT files for processing
    """
 
    def __init__(self, mission=None, verbose=False, evalluts=False,
                 timeout=10, clobber=False, dry_run=False):
 
        self.mission = mission
        self.verbose = verbose
        self.evalluts = evalluts
        self.timeout = timeout
        self.clobber = clobber
        self.dry_run = dry_run
        self.status = 0
        self.site_root = 'https://oceandata.sci.gsfc.nasa.gov/Ancillary/LUTs'
        self.localroot = os.getenv('OCVARROOT')
        self.sensor = SensorUtils.by_desc(mission)
 
        self.session = Session.SessionUtils(timeout=timeout, clobber=clobber, verbose=verbose)
 
    def lut_dirs(self):
        dirs = []
 
        # add instrument dir for MODIS utcpole.dat and leapsec.dat
        if self.sensor['instrument'] == 'MODIS':
            dirs.append(self.sensor['dir'])
 
        # add unique sensor name
        sensor = self.sensor['sensor'].lower()
        dirs.append(sensor)
 
        # add calibration dirs for MODIS, VIIRS and OCI
        if self.sensor['instrument'] in ['MODIS', 'VIIRS', 'OCI']:
            caldirs = ['cal', 'xcal']
            for caldir in caldirs:
                dirs.append(os.path.join(sensor, caldir, 'OPER'))
                if self.evalluts:
                    dirs.append(os.path.join(sensor, caldir, 'EVAL'))
 
        '''
        # Uncomment this section if we ever reorganize $OCVARROOT to match $OCDATAROOT
        dirs.append(self.sensor['dir'])
        subdir = self.sensor.get('subdir')
        if subdir:
            dirs.append(os.path.join(self.sensor['dir'], subdir))
        '''
        return dirs
 
    def get_luts(self):
 
        # regex for all valid suffixes
        # suffix = '\.(hdf|h5|nc|dat|txt)$'
        suffix = ''  # take whatever's there
        query = '?format=json'
 
        downloaded = []
        for d in self.lut_dirs():
            url = os.path.join(self.site_root, d, '') + query
            dirpath = os.path.join(self.localroot, d)
            if self.verbose:
                print()
                print('Downloading files into ' + dirpath)
 
            newfiles = []
            if self.mission == "common":
                newfiles = self.session.download_allfiles(
                    url, dirpath,
                    dry_run=self.dry_run, clobber=self.clobber,
                    regex="(?m)^(?!.*(?:gbr100|gbrReflectance|LandWater15ARC|VIIRS_DARKTARGET_LUT)).*$", 
                    check_times=True)
                if self.session.status:
                    self.status = 1
            else:
                # check times for non-versioned files
                new1 = self.session.download_allfiles(
                    url, dirpath,
                    dry_run=self.dry_run, clobber=self.clobber,
                    regex='^((?!\d+).)*' + suffix, check_times=True)
                if self.session.status:
                    self.status = 1
    
                # check only filesize for others
                new2 = self.session.download_allfiles(
                    url, dirpath,
                    dry_run=self.dry_run, clobber=self.clobber,
                    regex=suffix, check_times=False)
                if self.session.status:
                    self.status = 1
    
                newfiles = new1 + new2
 
 
            if len(newfiles) == 0:
                if self.verbose:
                    print('...no new files.')
            else:
                downloaded.append(newfiles)
 
                # remove outdated LUTs from OPER
                if 'OPER' in d:
                    purge_luts(newfiles, verbose=self.verbose)
 
        return downloaded
 
# end of class LutUtils