multilevel_processor Namespace Reference

Classes
class	ProcessorConfig
class	Sensor
class	Sensor_goci
class	Sensor_hawkeye
class	Sensor_meris
class	Sensor_modis
class	Sensor_seawifs
class	Sensor_viirs

Functions
def	get_obpg_data_file_object (file_specification)
def	build_executable_path (prog_name)
def	build_file_list_file (filename, file_list)
def	build_l2gen_par_file (par_contents, input_file, geo_file, output_file)
def	check_options (options)
def	clean_files (delete_list)
def	create_levels_list (rules_sets)
def	create_help_message (rules_sets)
def	do_processing (sensors_sets, par_file, cmd_line_ifile=None)
def	execute_command (command)
def	extract_par_section (par_contents, section)
def	find_geo_file (inp_file)
def	find_geo_file2 (inp_file, instrument, lvl)
def	find_viirs_geo_file (proc, first_svm_file)
def	get_batch_output_name (file_set, suffix)
def	get_data_file_option (par_contents, opt_text)
def	get_extract_params (proc)
def	get_file_date (filename)
def	get_file_handling_opts (par_contents)
def	get_input_files (par_data)
def	get_input_files_type_data (input_files_list)
def	get_intermediate_processors (sensor, existing_procs, rules, lowest_source_level)
def	get_intermediate_products (existing_prod_names, ruleset, lowest_source_level)
def	get_l2_extension ()
def	get_l3bin_extension ()
def	get_lowest_source_level (source_files)
def	get_options (par_data)
def	get_output_name2 (inp_files, targ_prog, suite=None, oformt=None, res=None)
def	get_output_name3 (input_name, input_files, suffix)
def	get_par_file_contents (par_file, acceptable_single_keys)
def	get_processors2 (sensor, par_contents, rules, lowest_source_level)
def	exe_processor (proc, src_files, src_lvl)
def	get_processors (src_files, input_files, par_contents, files_to_delete)
def	get_required_programs (target_program, ruleset, lowest_source_level)
def	get_source_geo_files (source_files, proc_src_types, proc_src_ndx)
def	get_source_file_sets (proc_src_types, source_files, src_key, requires_all_sources)
def	get_source_files (input_files)
def	get_source_products_types (targt_prod, ruleset)
def	get_traceback_message ()
def	initialze_sensors ()
def	log_and_exit (error_msg)
def	main ()
def	process_command_line (cl_parser)
def	read_file_list_file (flf_name)
def	run_batch_processor (processor, file_set)
def	run_nonbatch_processor (processor)
def	run_script (proc, script_name)
def	start_logging (time_stamp)
def	uniqify_list (orig_list)

Variables
bool	DEBUG = False
	cfg_data = None
list	FILE_USE_OPTS = ['deletefiles', 'overwrite', 'use_existing']
dictionary	SUFFIXES
dictionary	input_file_data = {}
	OCSSWROOT_DIR = os.environ['OCSSWROOT']

Detailed Description

Program to perform multilevel processing (previously known as the
seadas_processor and sometimes referred to as the 'uber' processor).

Function Documentation

build_executable_path()

def multilevel_processor.build_executable_path

(

prog_name

)

Returns the directory in which the program named in prog_name is found.
None is returned if the program is not found.

Definition at line 845 of file multilevel_processor.py.

build_file_list_file()

def multilevel_processor.build_file_list_file	(		filename,
			file_list
	)

Create a file listing the names of the files to be processed.

Definition at line 859 of file multilevel_processor.py.

build_l2gen_par_file()

def multilevel_processor.build_l2gen_par_file	(		par_contents,
			input_file,
			geo_file,
			output_file
	)

Build the parameter file for L2 processing.

Definition at line 867 of file multilevel_processor.py.

check_options()

def multilevel_processor.check_options

(

options

)

Check command line options

Definition at line 887 of file multilevel_processor.py.

clean_files()

def multilevel_processor.clean_files

(

delete_list

)

Delete unwanted files created during processing.

Definition at line 902 of file multilevel_processor.py.

create_help_message()

def multilevel_processor.create_help_message

(

rules_sets

)

Creates the message to be displayed when help is provided.

Definition at line 964 of file multilevel_processor.py.

create_levels_list()

def multilevel_processor.create_levels_list

(

rules_sets

)

Returns a list containing all the levels from all the rules sets.

Definition at line 942 of file multilevel_processor.py.

do_processing()

def multilevel_processor.do_processing	(		sensors_sets,
			par_file,
			cmd_line_ifile = None
	)

Perform the processing for each step (element of processor_list) needed.

Definition at line 1018 of file multilevel_processor.py.

exe_processor()

def multilevel_processor.exe_processor	(		proc,
			src_files,
			src_lvl
	)

Execute the processor.

Definition at line 1595 of file multilevel_processor.py.

execute_command()

def multilevel_processor.execute_command

(

command

)

Execute what is contained in command and then output the results to log
files and the console, as appropriate.

Definition at line 1103 of file multilevel_processor.py.

extract_par_section()

def multilevel_processor.extract_par_section	(		par_contents,
			section
	)

Returns a single section (e.g. L1a, GEO, L1B, L2, etc.) from the "par" file.

Definition at line 1124 of file multilevel_processor.py.

find_geo_file()

def multilevel_processor.find_geo_file

(

inp_file

)

Searches for a GEO file corresponding to inp_file.  If that GEO file exists,
returns that file name; otherwise, returns None.

Definition at line 1133 of file multilevel_processor.py.

find_geo_file2()

def multilevel_processor.find_geo_file2	(		inp_file,
			instrument,
			lvl
	)

Searches for a GEO file corresponding to inp_file.  If that GEO file exists,
returns that file name; otherwise, returns None.

Definition at line 1148 of file multilevel_processor.py.

find_viirs_geo_file()

def multilevel_processor.find_viirs_geo_file	(		proc,
			first_svm_file
	)

Searches for a GEO file corresponding to first_svm_file.  If that GEO file
exists, returns that file name; otherwise, returns None.

Definition at line 1185 of file multilevel_processor.py.

get_batch_output_name()

def multilevel_processor.get_batch_output_name	(		file_set,
			suffix
	)

Returns the output file for a "batch" run, i.e. a process that can accept
multiple inputs, such as l2bin or l3bin.

Definition at line 1195 of file multilevel_processor.py.

get_data_file_option()

def multilevel_processor.get_data_file_option	(		par_contents,
			opt_text
	)

If found in par_contents, the value for the option specified by opt_text
is returned; otherwise, False is returned.

Definition at line 1233 of file multilevel_processor.py.

get_extract_params()

def multilevel_processor.get_extract_params

(

proc

)

Run the lonlat2pixline program and return the parameters found.

Definition at line 1244 of file multilevel_processor.py.

get_file_date()

def multilevel_processor.get_file_date

(

filename

)

Get a Python Date object from a recognized file name's year and day of year.

Definition at line 1281 of file multilevel_processor.py.

get_file_handling_opts()

def multilevel_processor.get_file_handling_opts

(

par_contents

)

Returns the values of the file handling options in par_contents.

Definition at line 1304 of file multilevel_processor.py.

get_input_files()

def multilevel_processor.get_input_files

(

par_data

)

Get input files found in the uber par file's ifile line, a file list file,
or both.  Ensure that the list contains no duplicates.

Definition at line 1313 of file multilevel_processor.py.

get_input_files_type_data()

def multilevel_processor.get_input_files_type_data

(

input_files_list

)

Returns a dictionary with the the file_type (L0, L1A, L2, etc) and
instrument for each file in the input list.

Definition at line 1340 of file multilevel_processor.py.

get_intermediate_processors()

def multilevel_processor.get_intermediate_processors	(		sensor,
			existing_procs,
			rules,
			lowest_source_level
	)

Create processor objects for products which are needed, but not explicitly
specified in the par file.

Definition at line 1382 of file multilevel_processor.py.

get_intermediate_products()

def multilevel_processor.get_intermediate_products	(		existing_prod_names,
			ruleset,
			lowest_source_level
	)

Find products which are needed, but not explicitly specified by the
par file.

Definition at line 1400 of file multilevel_processor.py.

get_l2_extension()

def multilevel_processor.get_l2_extension

(

)

Returns the extension for an L2 file.  For the time being, this is
just '.L2'; however, different extensions may be wanted in the future, thus
this function is in place.

Definition at line 1417 of file multilevel_processor.py.

get_l3bin_extension()

def multilevel_processor.get_l3bin_extension

(

)

Returns the extension for an L3 Binned file.  For the time being, this is
just '.L3bin'; however, different extensions may be wanted in the future,
thus this function is in place.

Definition at line 1425 of file multilevel_processor.py.

get_lowest_source_level()

def multilevel_processor.get_lowest_source_level

(

source_files

)

Find the level of the lowest level source file to be processed.

Definition at line 1433 of file multilevel_processor.py.

get_obpg_data_file_object()

def multilevel_processor.get_obpg_data_file_object

(

file_specification

)

Returns an obpg_data_file object for the file named in file_specification.

Definition at line 833 of file multilevel_processor.py.

get_options()

def multilevel_processor.get_options

(

par_data

)

Extract the options for a program to be run from the corresponding data in
the uber par file.

Definition at line 1449 of file multilevel_processor.py.

get_output_name2()

def multilevel_processor.get_output_name2	(		inp_files,
			targ_prog,
			suite = None,
			oformt = None,
			res = None
	)

Determine what the output name would be if targ_prog is run on input_files.

Definition at line 1463 of file multilevel_processor.py.

get_output_name3()

def multilevel_processor.get_output_name3	(		input_name,
			input_files,
			suffix
	)

Determine the output name for a program to be run.

Definition at line 1480 of file multilevel_processor.py.

get_par_file_contents()

def multilevel_processor.get_par_file_contents	(		par_file,
			acceptable_single_keys
	)

Return the contents of the input "par" file.

Definition at line 1522 of file multilevel_processor.py.

get_processors()

def multilevel_processor.get_processors	(		src_files,
			input_files,
			par_contents,
			files_to_delete
	)

Determine how to chain the processors together.

Definition at line 1627 of file multilevel_processor.py.

get_processors2()

def multilevel_processor.get_processors2	(		sensor,
			par_contents,
			rules,
			lowest_source_level
	)

Determine the processors which are needed.

Definition at line 1577 of file multilevel_processor.py.

get_required_programs()

def multilevel_processor.get_required_programs	(		target_program,
			ruleset,
			lowest_source_level
	)

Returns the programs required too produce the desired final output.

Definition at line 2008 of file multilevel_processor.py.

get_source_file_sets()

def multilevel_processor.get_source_file_sets	(		proc_src_types,
			source_files,
			src_key,
			requires_all_sources
	)

Returns the set of source files needed.

Definition at line 2051 of file multilevel_processor.py.

get_source_files()

def multilevel_processor.get_source_files

(

input_files

)

Returns a dictionary containing the programs to be run (as keys) and the
a list of files on which that program should be run.

Definition at line 2110 of file multilevel_processor.py.

get_source_geo_files()

def multilevel_processor.get_source_geo_files	(		source_files,
			proc_src_types,
			proc_src_ndx
	)

:param source_files: list of source files
:param proc_src_types: list of source types for the processor
:param proc_src_ndx: index into the proc_src_types list pointing to the
                     source type to use to get the input files
:return: list of GEO files that correspond to the files in source_files

Definition at line 2031 of file multilevel_processor.py.

get_source_products_types()

def multilevel_processor.get_source_products_types	(		targt_prod,
			ruleset
	)

Return the list of source product types needed to produce the final product.

Definition at line 2124 of file multilevel_processor.py.

get_traceback_message()

def multilevel_processor.get_traceback_message

(

)

Returns an error message built from traceback data.

Definition at line 2139 of file multilevel_processor.py.

initialze_sensors()

def multilevel_processor.initialze_sensors

(

)

Initialize sensors.

Definition at line 2156 of file multilevel_processor.py.

log_and_exit()

def multilevel_processor.log_and_exit

(

error_msg

)

Record error_msg in the debug log, then exit with error_msg going to stderr
and an exit code of 1; see:
    http://docs.python.org/library/sys.html#exit.

Definition at line 2169 of file multilevel_processor.py.

main()

def multilevel_processor.main

(

)

main processing function.

Definition at line 2178 of file multilevel_processor.py.

process_command_line()

def multilevel_processor.process_command_line

(

cl_parser

)

Get arguments and options from the calling command line.
To be consistent with other OBPG programs, an underscore ('_') is used for
multiword options, instead of a dash ('-').

Definition at line 2243 of file multilevel_processor.py.

read_file_list_file()

def multilevel_processor.read_file_list_file

(

flf_name

)

Reads flf_name and returns the list of files to be processed.

Definition at line 2283 of file multilevel_processor.py.

run_batch_processor()

def multilevel_processor.run_batch_processor	(		processor,
			file_set
	)

Run a processor, e.g. l2bin, which processes batches of files.

Definition at line 2304 of file multilevel_processor.py.

run_nonbatch_processor()

def multilevel_processor.run_nonbatch_processor

(

processor

)

Run a processor which deals with single input files (or pairs of files in
the case of MODIS L1B processing in which GEO files are also needed).

Definition at line 2352 of file multilevel_processor.py.

run_script()

def multilevel_processor.run_script	(		proc,
			script_name
	)

Build the command to run the processing script which is passed in.

Definition at line 2428 of file multilevel_processor.py.

start_logging()

def multilevel_processor.start_logging

(

time_stamp

)

Opens log file(s) for debugging.

Definition at line 2443 of file multilevel_processor.py.

uniqify_list()

def multilevel_processor.uniqify_list

(

orig_list

)

Returns a list with no duplicates.  Somewhat borrowed from:
  http://www.peterbe.com/plog/uniqifiers-benchmark (example f5)

Definition at line 2466 of file multilevel_processor.py.

Variable Documentation

cfg_data

cfg_data = None

Definition at line 2484 of file multilevel_processor.py.

DEBUG

bool DEBUG = False

Definition at line 2481 of file multilevel_processor.py.

FILE_USE_OPTS

list FILE_USE_OPTS = ['deletefiles', 'overwrite', 'use_existing']

Definition at line 2485 of file multilevel_processor.py.

input_file_data

dictionary input_file_data = {}

Definition at line 2505 of file multilevel_processor.py.

OCSSWROOT_DIR

OCSSWROOT_DIR = os.environ['OCSSWROOT']

Definition at line 2509 of file multilevel_processor.py.

SUFFIXES

dictionary SUFFIXES

Initial value:

=  {
    'geo': 'GEO',
    'l1brsgen': 'L1B_BRS',
    'l1aextract': 'L1A.sub',
    'l1aextract_viirs': 'L1A.sub',
    'l1aextract_seawifs': 'L1A.sub',
    'l1aextract_modis': 'L1A.sub',
    # 'l1mapgen': 'L1B_MAP',
    'l2bin': 'L3b',
    'l2brsgen': 'L2_BRS',
    'l2extract': 'L2.sub',
    'l2gen': 'L2',
    # 'l2mapgen': 'L2B_MAP',
    'l3bin': 'L3b',
    'l3mapgen': 'L3m',
    'level 1a': 'L1A',
    'level 1b': 'L1B_LAC',
    # 'smigen': 'SMI'
}

Definition at line 2486 of file multilevel_processor.py.