Repository navigation
Issue while using multiprocessing #169
Description
Activity
Can you share a full code to test on my end?
Some code is missing from the posted example.
import pygad import time def fitness_func(ga_instance, solution, solution_idx): #cal ISP for each gamma in population => ehanced image conf = Config_file_handling.YAMLfunctions() config = conf.read_yaml('Default_isp_conf.yml') config['gac']['gamma'] = float(solution[0]) config['nlm']['h'] = int(solution[1]) conf.write_yaml('Default_isp_conf.yml', config) config = conf.read_yaml('config.yml') IQM = ISP(config) iqm = mean(IQM) return round(iqm['PSNR'],4) ga_instance = pygad.GA(num_generations=10, num_parents_mating=2, sol_per_pop=20, num_genes=2, gene_type=gene_type, gene_space=gene_space, fitness_func=fitness_func, on_generation=on_generation, parent_selection_type="rank", keep_parents=1, crossover_type='single_point', crossover_probability=0.8, mutation_type='random', mutation_probability=0.2, allow_duplicate_genes=False, #stop_criteria="saturate_5", save_best_solutions=True, save_solutions=True, parallel_processing=['process', 5]) t1 = time.time() ga_instance.run() t2 = time.time() print("Time is", t2-t1)
import pygad import time def mean(IQM): mean_dict = {} for key in IQM[0].keys(): mean_dict[key] = sum(d[key] for d in IQM) / len(IQM) return mean_dict def fitness_func(ga_instance, solution, solution_idx): #cal ISP for each gamma in population => ehanced image conf = Config_file_handling.YAMLfunctions() config = conf.read_yaml('Default_isp_conf.yml') config['gac']['gamma'] = float(solution[0]) config['nlm']['h'] = int(solution[1]) conf.write_yaml('Default_isp_conf.yml', config) config = conf.read_yaml('config.yml') IQM = ISP(config) iqm = mean(IQM) return round(iqm['PSNR'],4) ga_instance = pygad.GA(num_generations=10, num_parents_mating=2, sol_per_pop=20, num_genes=2, gene_type=gene_type, gene_space=gene_space, fitness_func=fitness_func, on_generation=on_generation, parent_selection_type="rank", keep_parents=1, crossover_type='single_point', crossover_probability=0.8, mutation_type='random', mutation_probability=0.2, allow_duplicate_genes=False, #stop_criteria="saturate_5", save_best_solutions=True, save_solutions=True, parallel_processing=['process', 5]) t1 = time.time() ga_instance.run() t2 = time.time() print("Time is", t2-t1)Main:
`def image_quality_matrices(original_img, modified_img) -> dict:
"""This functions handles the IQM in main"""iqm = IQM.ImageQualityMatrices(original_img) iqm_dict = {'PSNR': iqm.PSNR(modified_img), 'SSIM': iqm.SSIM(modified_img), 'UQI': iqm.uqi(P=modified_img)} #print(iqm_dict) return iqm_dictdef ISP(conf):
""" Executes the Fast Open ISP in main """
list_iqms = []
# Create Log File directory if not available
os.makedirs(conf['log_file_directory'], exist_ok=True)# Checks the input Directory if not os.path.isdir(conf['images_input_directory']): print("Please specify the correct input path") sys.exit() else: for image in glob.glob(conf['images_input_directory'] + '*'): log_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S,') isp_conf = Config_file_handling.YAMLfunctions() isp_params = isp_conf.read_yaml(conf['configuration_file_path']) file_name = image.split('\\')[-1].split('.')[0] print(file_name) try: # Parameters Optimization # optimize = Parameters_optimization.Optimization(image) # optimize.param_optimize(conf['configuration_file_path']) # Getting the image size from metadata Image_metadata.metadata_conf(image, conf['configuration_file_path']) # Running the ISP isp = isp_run.RunISPs(image_file=image, output_dir=conf['images_output_directory'], add_fileTime=conf[ 'add_time_along_with_FileName']) ISPOut_dict = isp.runISP(conf['configuration_file_path'], isp_name=conf['available_ISPs'][conf[ "ISP_to_use"]-1]) if conf['Perform IQM Checks']['status']: # Getting the Path of Ground Truth Image file_name = image.split('\\')[-1] ext = file_name.split('.')[-1] o_image_name = file_name.replace(ext, 'jpg') o_image_path = f'{conf["Perform IQM Checks"]["ground_truth_directory"]}\\{o_image_name}' # Performing Image Quality Checks iqms = image_quality_matrices(o_image_path, ISPOut_dict['OutputImage']) #up_iqms = {"Image": file_name} #up_iqms.update(iqms) list_iqms.append(iqms) print(list_iqms) # Writing in log file with open(conf['log_file_directory']+conf['log_file_name'], 'a') as log_file: log_file.write(f'{log_time} INFO, {ISPOut_dict["FileName"]}, {iqms}, {isp_params}\n') else: with open(conf['log_file_directory']+conf['log_file_name'], 'a') as log_file: log_file.write(f'{log_time} INFO, {ISPOut_dict["FileName"]}, {isp_params}\n') except Exception as error: exc_type, exc_obj, exc_tb = sys.exc_info() fName = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1] template = "{0}, File {1}, Line {2}\n{3}" message = template.format(exc_type, fName, exc_tb.tb_lineno, error.args).replace('\n', ' ') with open(conf['log_file_directory']+conf['log_file_name'], 'a') as log_file: log_file.write(f'{log_time} {file_name}, {message}, {isp_params}\n') return list_iqms`Thanks for sharing the code.
Please provide the following to run the code.
Config_file_handlingisp_runImage_metadatagene_typegene_spaceon_generation
While using multiprocessing for my task my program gets the exception after some generations. Sometimes it works fine for all generations and stops working abruptly and generates "NoneType object is unsubscriptable" or Pool halt errors. I go through the documentation but there is no detail provided about how multiprocessing is working and what to do and what not to do while using this. Please guide me in this regard to what can be the possible issue. My code is
def fitness_func(solution, solution_idx):
#cal ISP for each gamma in population => ehanced image
conf = Config_file_handling.YAMLfunctions()
config = conf.read_yaml('Default_isp_conf.yml')
config['gac']['gamma'] = float(solution[0])
config['nlm']['h'] = int(solution[1])
conf.write_yaml('Default_isp_conf.yml', config)
config = conf.read_yaml('config.yml')
IQM = ISP(config)
iqm = mean(IQM)
return round(iqm['PSNR'],4)
Create the GA instance
ga_instance = pygad.GA(num_generations=10,
num_parents_mating=2,
sol_per_pop=20,
num_genes=2,
gene_type=gene_type,
gene_space=gene_space,
fitness_func=fitness_func,
on_generation=on_generation,
parent_selection_type="rank",
keep_parents=1,
crossover_type='single_point',
crossover_probability=0.8,
mutation_type='random',
mutation_probability=0.2,
allow_duplicate_genes=False,
#stop_criteria="saturate_5",
save_best_solutions=True,
save_solutions=True,
parallel_processing=['process', 5]
)
t1 = time.time()
Run the GA
ga_instance.run()
t2 = time.time()
print("Time is", t2-t1)