Using RailProject
This notebook will show you the basics using the RailProject class to manage an analysis project
Setup and teardown scripts to setup a test area
[1]:
import os
from rail.projects import library
check_dir = os.path.basename(os.path.abspath(os.curdir))
if check_dir == 'examples':
os.chdir('..')
setup = library.setup_project_area()
assert setup == 0
# use this to cleanup
# library.teardown_project_area()
/home/docs/checkouts/readthedocs.org/user_builds/rail-projects/envs/latest/lib/python3.11/site-packages/ceci/__init__.py:12: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.
from pkg_resources import DistributionNotFound
temp_data/
temp_data/data/
temp_data/data/ci_test_v1.1.3/
temp_data/data/ci_test_v1.1.3/9924/
temp_data/data/ci_test_v1.1.3/9924/part-0.parquet
temp_data/data/ci_test_v1.1.3/9925/
temp_data/data/ci_test_v1.1.3/9925/part-0.parquet
temp_data/projects/
temp_data/projects/ci_test/
temp_data/projects/ci_test/data/
temp_data/projects/ci_test/data/blend_baseline/
temp_data/projects/ci_test/data/blend_baseline/true_NZ_true_nz_knn_equal_count_bin2.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_equal_count_bin3_point_est_hist.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_estimate_trainz.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_uniform_binning_bin2_naive_stack.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_equal_count_bin0_naive_stack.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_equal_count_bin1_naive_stack.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_equal_count_bin2_naive_stack.hdf5
temp_data/projects/ci_test/data/blend_baseline/true_NZ_true_nz_knn_uniform_binning_bin3.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_evaluate_fzboost.hdf5
temp_data/projects/ci_test/data/blend_baseline/true_NZ_true_nz_knn_uniform_binning_bin4.hdf5
temp_data/projects/ci_test/data/blend_baseline/true_NZ_true_nz_knn_equal_count_bin1.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_equal_count_bin4_point_est_hist.hdf5
temp_data/projects/ci_test/data/blend_baseline/model_inform_simplenn.pkl
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_equal_count_bin0_point_est_hist.hdf5
temp_data/projects/ci_test/data/blend_baseline/true_NZ_true_nz_knn_equal_count_bin3.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_equal_count_bin4_naive_stack.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_equal_count_bin1_point_est_hist.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_uniform_binning_bin3_naive_stack.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_uniform_binning_bin1_point_est_hist.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_uniform_binning_bin1_naive_stack.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_estimate_knn.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_uniform_binning_bin4_point_est_hist.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_uniform_binning_bin0_point_est_hist.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_evaluate_knn.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_evaluate_trainz.hdf5
temp_data/projects/ci_test/data/blend_baseline/model_inform_gpz.pkl
temp_data/projects/ci_test/data/blend_baseline/true_NZ_true_nz_knn_uniform_binning_bin1.hdf5
temp_data/projects/ci_test/data/blend_baseline/true_NZ_true_nz_knn_uniform_binning_bin2.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_estimate_gpz.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_estimate_fzboost.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_estimate_bpz.hdf5
temp_data/projects/ci_test/data/blend_baseline/true_NZ_true_nz_knn_uniform_binning_bin0.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_uniform_binning_bin4_naive_stack.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_uniform_binning_bin2_point_est_hist.hdf5
temp_data/projects/ci_test/data/blend_baseline/model_inform_sompz.pkl
temp_data/projects/ci_test/data/blend_baseline/true_NZ_true_nz_knn_equal_count_bin0.hdf5
temp_data/projects/ci_test/data/blend_baseline/true_NZ_true_nz_knn_equal_count_bin4.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_estimate_simplenn.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_equal_count_bin2_point_est_hist.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_equal_count_bin3_naive_stack.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_evaluate_simplenn.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_uniform_binning_bin0_naive_stack.hdf5
temp_data/projects/ci_test/data/blend_baseline/single_NZ_summarize_knn_uniform_binning_bin3_point_est_hist.hdf5
temp_data/projects/ci_test/data/blend_baseline/output_evaluate_gpz.hdf5
temp_data/projects/ci_test/data/blend_baseline/model_inform_knn.pkl
temp_data/projects/ci_test/data/blend_baseline/model_inform_fzboost.pkl
temp_data/projects/ci_test/data/blend_baseline/model_inform_bpz.pkl
temp_data/projects/ci_test/data/blend_baseline/model_inform_train_z.pkl
temp_data/projects/ci_test/data/blend_baseline/model_inform_trainz.pkl
Load the test project
[2]:
from rail.projects import RailProject
project = RailProject.load_config("tests/ci_project.yaml")
Inspect the test project
[3]:
catalog_files_truth = project.get_catalog_files("truth")
print(catalog_files_truth)
['tests/temp_data/data/ci_test_v1.1.3/9924/part-0.parquet', 'tests/temp_data/data/ci_test_v1.1.3/9925/part-0.parquet']
Run a data reduction algorithm on the test project data
This will use the “roman_rubin” reducer to apply the “gold” selection to the “truth” catalog to make a “reduced” catalog
[4]:
project.reduce_data(
catalog_template="truth",
output_catalog_template="reduced",
reducer_class_name="roman_rubin",
input_selection="",
selection="gold",
)
writing dataset to tests/temp_data/data/ci_test_v1.1.3_gold/9924/part-0.pq
writing dataset to tests/temp_data/data/ci_test_v1.1.3_gold/9925/part-0.pq
[4]:
['tests/temp_data/data/ci_test_v1.1.3_gold/9924/part-0.pq',
'tests/temp_data/data/ci_test_v1.1.3_gold/9925/part-0.pq']
Subsample the test project
This will use the “random_subsampler” to apply the “train_10” subsample to the “reduced” catalog of the baseline flavor with the gold selection
[5]:
project.subsample_data(
catalog_template="reduced",
file_template="train_file_10",
subsampler_class_name="random_subsampler",
subsample_name="train_10",
flavor="baseline",
selection="gold",
)
num rows 238
sampling 100000
writing tests/temp_data/data/test/ci_test_gold_baseline_10.parquet
done
[5]:
'tests/temp_data/data/test/ci_test_gold_baseline_10.parquet'
Build all the pipelines for the “baseline” flavor
[6]:
project.build_pipelines(flavor="baseline")
Writing tests/temp_data/projects/ci_test/pipelines/truth_to_observed_baseline.yaml
Inserting handle into data store. output_unrec_bl: inprogress_output_unrec_bl.pq, unrec_bl
Inserting handle into data store. input: None, error_model_lsst
Inserting handle into data store. output_error_model_lsst: inprogress_output_error_model_lsst.pq, error_model_lsst
Inserting handle into data store. output_deredden_lsst: inprogress_output_deredden_lsst.pq, deredden_lsst
Inserting handle into data store. output_error_model_roman: inprogress_output_error_model_roman.pq, error_model_roman
Inserting handle into data store. output_deredden_roman: inprogress_output_deredden_roman.pq, deredden_roman
Writing tests/temp_data/projects/ci_test/pipelines/prepare_baseline.yaml
Inserting handle into data store. output_flux_to_mag: inprogress_output_flux_to_mag.pq, flux_to_mag
Writing tests/temp_data/projects/ci_test/pipelines/photometric_errors_baseline.yaml
Inserting handle into data store. output_reddener: inprogress_output_reddener.pq, reddener
Writing tests/temp_data/projects/ci_test/pipelines/blending_baseline.yaml
Writing tests/temp_data/projects/ci_test/pipelines/spec_selection_baseline.yaml
Writing tests/temp_data/projects/ci_test/pipelines/inform_baseline.yaml
Writing tests/temp_data/projects/ci_test/pipelines/estimate_baseline.yaml
Writing tests/temp_data/projects/ci_test/pipelines/evaluate_baseline.yaml
Writing tests/temp_data/projects/ci_test/pipelines/pz_baseline.yaml
Inserting handle into data store. model_inform_trainz: inprogress_model_inform_trainz.pkl, inform_trainz
Inserting handle into data store. model: None, estimate_trainz
Inserting handle into data store. output_estimate_trainz: inprogress_output_estimate_trainz.hdf5, estimate_trainz
Inserting handle into data store. model_inform_simplenn: inprogress_model_inform_simplenn.pkl, inform_simplenn
Inserting handle into data store. output_estimate_simplenn: inprogress_output_estimate_simplenn.hdf5, estimate_simplenn
Inserting handle into data store. model_inform_fzboost: inprogress_model_inform_fzboost.pkl, inform_fzboost
Inserting handle into data store. output_estimate_fzboost: inprogress_output_estimate_fzboost.hdf5, estimate_fzboost
Inserting handle into data store. model_inform_knn: inprogress_model_inform_knn.pkl, inform_knn
Inserting handle into data store. output_estimate_knn: inprogress_output_estimate_knn.hdf5, estimate_knn
Inserting handle into data store. model_inform_gpz: inprogress_model_inform_gpz.pkl, inform_gpz
Inserting handle into data store. output_estimate_gpz: inprogress_output_estimate_gpz.hdf5, estimate_gpz
Writing tests/temp_data/projects/ci_test/pipelines/inform_sompz_baseline.yaml
Writing tests/temp_data/projects/ci_test/pipelines/estimate_sompz_baseline.yaml
Inserting handle into data store. assignment_som_deepspec_estimator: inprogress_assignment_som_deepspec_estimator.hdf5, som_deepspec_estimator
Inserting handle into data store. cell_deep_spec_data: None, som_pzc
Inserting handle into data store. assignment_som_deepdeep_estimator: inprogress_assignment_som_deepdeep_estimator.hdf5, som_deepdeep_estimator
Inserting handle into data store. assignment_som_deepwide_estimator: inprogress_assignment_som_deepwide_estimator.hdf5, som_deepwide_estimator
Inserting handle into data store. cell_deep_balrog_data: None, som_pcchat
Inserting handle into data store. cell_wide_balrog_data: None, som_pcchat
Inserting handle into data store. assignment_som_widewide_estimator: inprogress_assignment_som_widewide_estimator.hdf5, som_widewide_estimator
Inserting handle into data store. pz_c_som_pzc: inprogress_pz_c_som_pzc.hdf5, som_pzc
Inserting handle into data store. pc_chat_som_pcchat: inprogress_pc_chat_som_pcchat.hdf5, som_pcchat
Inserting handle into data store. cell_wide_wide_data: None, som_pzchat
Inserting handle into data store. pz_c: None, som_pzchat
Inserting handle into data store. pc_chat: None, som_pzchat
Inserting handle into data store. assignment_som_widespec_estimator: inprogress_assignment_som_widespec_estimator.hdf5, som_widespec_estimator
Inserting handle into data store. cell_wide_spec_data: None, som_tomobin
Inserting handle into data store. tomo_bins_wide_som_tomobin: inprogress_tomo_bins_wide_som_tomobin.hdf5, som_tomobin
Inserting handle into data store. tomo_bins_wide: None, som_nz
Writing tests/temp_data/projects/ci_test/pipelines/inform_somlike_baseline.yaml
Writing tests/temp_data/projects/ci_test/pipelines/somlike_recalib_baseline.yaml
Inserting handle into data store. output_pz_deepdeep_estimator_trainz: inprogress_output_pz_deepdeep_estimator_trainz.hdf5, pz_deepdeep_estimator_trainz
Inserting handle into data store. pz_estimate: None, deepdeep_assigment_trainz_pz_mode
Inserting handle into data store. output_pz_deepspec_estimator_trainz: inprogress_output_pz_deepspec_estimator_trainz.hdf5, pz_deepspec_estimator_trainz
Inserting handle into data store. output_pz_deepwide_estimator_trainz: inprogress_output_pz_deepwide_estimator_trainz.hdf5, pz_deepwide_estimator_trainz
Inserting handle into data store. output_pz_widewide_estimator_trainz: inprogress_output_pz_widewide_estimator_trainz.hdf5, pz_widewide_estimator_trainz
Inserting handle into data store. output_pz_widespec_estimator_trainz: inprogress_output_pz_widespec_estimator_trainz.hdf5, pz_widespec_estimator_trainz
Inserting handle into data store. assignment_deepspec_assigment_trainz_pz_mode: inprogress_assignment_deepspec_assigment_trainz_pz_mode.hdf5, deepspec_assigment_trainz_pz_mode
Inserting handle into data store. assignment_deepdeep_assigment_trainz_pz_mode: inprogress_assignment_deepdeep_assigment_trainz_pz_mode.hdf5, deepdeep_assigment_trainz_pz_mode
Inserting handle into data store. assignment_deepwide_assigment_trainz_pz_mode: inprogress_assignment_deepwide_assigment_trainz_pz_mode.hdf5, deepwide_assigment_trainz_pz_mode
Inserting handle into data store. assignment_widewide_assigment_trainz_pz_mode: inprogress_assignment_widewide_assigment_trainz_pz_mode.hdf5, widewide_assigment_trainz_pz_mode
Inserting handle into data store. pz_c_som_pzc_trainz_pz_mode: inprogress_pz_c_som_pzc_trainz_pz_mode.hdf5, som_pzc_trainz_pz_mode
Inserting handle into data store. pc_chat_som_pcchat_trainz_pz_mode: inprogress_pc_chat_som_pcchat_trainz_pz_mode.hdf5, som_pcchat_trainz_pz_mode
Inserting handle into data store. assignment_widespec_assigment_trainz_pz_mode: inprogress_assignment_widespec_assigment_trainz_pz_mode.hdf5, widespec_assigment_trainz_pz_mode
Inserting handle into data store. tomo_bins_wide_som_tomobin_trainz_pz_mode: inprogress_tomo_bins_wide_som_tomobin_trainz_pz_mode.hdf5, som_tomobin_trainz_pz_mode
Inserting handle into data store. assignment_deepspec_assigment_trainz_pz_max_cell_p: inprogress_assignment_deepspec_assigment_trainz_pz_max_cell_p.hdf5, deepspec_assigment_trainz_pz_max_cell_p
Inserting handle into data store. assignment_deepdeep_assigment_trainz_pz_max_cell_p: inprogress_assignment_deepdeep_assigment_trainz_pz_max_cell_p.hdf5, deepdeep_assigment_trainz_pz_max_cell_p
Inserting handle into data store. assignment_deepwide_assigment_trainz_pz_max_cell_p: inprogress_assignment_deepwide_assigment_trainz_pz_max_cell_p.hdf5, deepwide_assigment_trainz_pz_max_cell_p
Inserting handle into data store. assignment_widewide_assigment_trainz_pz_max_cell_p: inprogress_assignment_widewide_assigment_trainz_pz_max_cell_p.hdf5, widewide_assigment_trainz_pz_max_cell_p
Inserting handle into data store. pz_c_som_pzc_trainz_pz_max_cell_p: inprogress_pz_c_som_pzc_trainz_pz_max_cell_p.hdf5, som_pzc_trainz_pz_max_cell_p
Inserting handle into data store. pc_chat_som_pcchat_trainz_pz_max_cell_p: inprogress_pc_chat_som_pcchat_trainz_pz_max_cell_p.hdf5, som_pcchat_trainz_pz_max_cell_p
Inserting handle into data store. assignment_widespec_assigment_trainz_pz_max_cell_p: inprogress_assignment_widespec_assigment_trainz_pz_max_cell_p.hdf5, widespec_assigment_trainz_pz_max_cell_p
Inserting handle into data store. tomo_bins_wide_som_tomobin_trainz_pz_max_cell_p: inprogress_tomo_bins_wide_som_tomobin_trainz_pz_max_cell_p.hdf5, som_tomobin_trainz_pz_max_cell_p
Inserting handle into data store. output_pz_deepdeep_estimator_simplenn: inprogress_output_pz_deepdeep_estimator_simplenn.hdf5, pz_deepdeep_estimator_simplenn
Inserting handle into data store. output_pz_deepspec_estimator_simplenn: inprogress_output_pz_deepspec_estimator_simplenn.hdf5, pz_deepspec_estimator_simplenn
Inserting handle into data store. output_pz_deepwide_estimator_simplenn: inprogress_output_pz_deepwide_estimator_simplenn.hdf5, pz_deepwide_estimator_simplenn
Inserting handle into data store. output_pz_widewide_estimator_simplenn: inprogress_output_pz_widewide_estimator_simplenn.hdf5, pz_widewide_estimator_simplenn
Inserting handle into data store. output_pz_widespec_estimator_simplenn: inprogress_output_pz_widespec_estimator_simplenn.hdf5, pz_widespec_estimator_simplenn
Inserting handle into data store. assignment_deepspec_assigment_simplenn_pz_mode: inprogress_assignment_deepspec_assigment_simplenn_pz_mode.hdf5, deepspec_assigment_simplenn_pz_mode
Inserting handle into data store. assignment_deepdeep_assigment_simplenn_pz_mode: inprogress_assignment_deepdeep_assigment_simplenn_pz_mode.hdf5, deepdeep_assigment_simplenn_pz_mode
Inserting handle into data store. assignment_deepwide_assigment_simplenn_pz_mode: inprogress_assignment_deepwide_assigment_simplenn_pz_mode.hdf5, deepwide_assigment_simplenn_pz_mode
Inserting handle into data store. assignment_widewide_assigment_simplenn_pz_mode: inprogress_assignment_widewide_assigment_simplenn_pz_mode.hdf5, widewide_assigment_simplenn_pz_mode
Inserting handle into data store. pz_c_som_pzc_simplenn_pz_mode: inprogress_pz_c_som_pzc_simplenn_pz_mode.hdf5, som_pzc_simplenn_pz_mode
Inserting handle into data store. pc_chat_som_pcchat_simplenn_pz_mode: inprogress_pc_chat_som_pcchat_simplenn_pz_mode.hdf5, som_pcchat_simplenn_pz_mode
Inserting handle into data store. assignment_widespec_assigment_simplenn_pz_mode: inprogress_assignment_widespec_assigment_simplenn_pz_mode.hdf5, widespec_assigment_simplenn_pz_mode
Inserting handle into data store. tomo_bins_wide_som_tomobin_simplenn_pz_mode: inprogress_tomo_bins_wide_som_tomobin_simplenn_pz_mode.hdf5, som_tomobin_simplenn_pz_mode
Inserting handle into data store. assignment_deepspec_assigment_simplenn_pz_max_cell_p: inprogress_assignment_deepspec_assigment_simplenn_pz_max_cell_p.hdf5, deepspec_assigment_simplenn_pz_max_cell_p
Inserting handle into data store. assignment_deepdeep_assigment_simplenn_pz_max_cell_p: inprogress_assignment_deepdeep_assigment_simplenn_pz_max_cell_p.hdf5, deepdeep_assigment_simplenn_pz_max_cell_p
Inserting handle into data store. assignment_deepwide_assigment_simplenn_pz_max_cell_p: inprogress_assignment_deepwide_assigment_simplenn_pz_max_cell_p.hdf5, deepwide_assigment_simplenn_pz_max_cell_p
Inserting handle into data store. assignment_widewide_assigment_simplenn_pz_max_cell_p: inprogress_assignment_widewide_assigment_simplenn_pz_max_cell_p.hdf5, widewide_assigment_simplenn_pz_max_cell_p
Inserting handle into data store. pz_c_som_pzc_simplenn_pz_max_cell_p: inprogress_pz_c_som_pzc_simplenn_pz_max_cell_p.hdf5, som_pzc_simplenn_pz_max_cell_p
Inserting handle into data store. pc_chat_som_pcchat_simplenn_pz_max_cell_p: inprogress_pc_chat_som_pcchat_simplenn_pz_max_cell_p.hdf5, som_pcchat_simplenn_pz_max_cell_p
Inserting handle into data store. assignment_widespec_assigment_simplenn_pz_max_cell_p: inprogress_assignment_widespec_assigment_simplenn_pz_max_cell_p.hdf5, widespec_assigment_simplenn_pz_max_cell_p
Inserting handle into data store. tomo_bins_wide_som_tomobin_simplenn_pz_max_cell_p: inprogress_tomo_bins_wide_som_tomobin_simplenn_pz_max_cell_p.hdf5, som_tomobin_simplenn_pz_max_cell_p
Inserting handle into data store. output_pz_deepdeep_estimator_fzboost: inprogress_output_pz_deepdeep_estimator_fzboost.hdf5, pz_deepdeep_estimator_fzboost
Inserting handle into data store. output_pz_deepspec_estimator_fzboost: inprogress_output_pz_deepspec_estimator_fzboost.hdf5, pz_deepspec_estimator_fzboost
Inserting handle into data store. output_pz_deepwide_estimator_fzboost: inprogress_output_pz_deepwide_estimator_fzboost.hdf5, pz_deepwide_estimator_fzboost
Inserting handle into data store. output_pz_widewide_estimator_fzboost: inprogress_output_pz_widewide_estimator_fzboost.hdf5, pz_widewide_estimator_fzboost
Inserting handle into data store. output_pz_widespec_estimator_fzboost: inprogress_output_pz_widespec_estimator_fzboost.hdf5, pz_widespec_estimator_fzboost
Inserting handle into data store. assignment_deepspec_assigment_fzboost_pz_mode: inprogress_assignment_deepspec_assigment_fzboost_pz_mode.hdf5, deepspec_assigment_fzboost_pz_mode
Inserting handle into data store. assignment_deepdeep_assigment_fzboost_pz_mode: inprogress_assignment_deepdeep_assigment_fzboost_pz_mode.hdf5, deepdeep_assigment_fzboost_pz_mode
Inserting handle into data store. assignment_deepwide_assigment_fzboost_pz_mode: inprogress_assignment_deepwide_assigment_fzboost_pz_mode.hdf5, deepwide_assigment_fzboost_pz_mode
Inserting handle into data store. assignment_widewide_assigment_fzboost_pz_mode: inprogress_assignment_widewide_assigment_fzboost_pz_mode.hdf5, widewide_assigment_fzboost_pz_mode
Inserting handle into data store. pz_c_som_pzc_fzboost_pz_mode: inprogress_pz_c_som_pzc_fzboost_pz_mode.hdf5, som_pzc_fzboost_pz_mode
Inserting handle into data store. pc_chat_som_pcchat_fzboost_pz_mode: inprogress_pc_chat_som_pcchat_fzboost_pz_mode.hdf5, som_pcchat_fzboost_pz_mode
Inserting handle into data store. assignment_widespec_assigment_fzboost_pz_mode: inprogress_assignment_widespec_assigment_fzboost_pz_mode.hdf5, widespec_assigment_fzboost_pz_mode
Inserting handle into data store. tomo_bins_wide_som_tomobin_fzboost_pz_mode: inprogress_tomo_bins_wide_som_tomobin_fzboost_pz_mode.hdf5, som_tomobin_fzboost_pz_mode
Inserting handle into data store. assignment_deepspec_assigment_fzboost_pz_max_cell_p: inprogress_assignment_deepspec_assigment_fzboost_pz_max_cell_p.hdf5, deepspec_assigment_fzboost_pz_max_cell_p
Inserting handle into data store. assignment_deepdeep_assigment_fzboost_pz_max_cell_p: inprogress_assignment_deepdeep_assigment_fzboost_pz_max_cell_p.hdf5, deepdeep_assigment_fzboost_pz_max_cell_p
Inserting handle into data store. assignment_deepwide_assigment_fzboost_pz_max_cell_p: inprogress_assignment_deepwide_assigment_fzboost_pz_max_cell_p.hdf5, deepwide_assigment_fzboost_pz_max_cell_p
Inserting handle into data store. assignment_widewide_assigment_fzboost_pz_max_cell_p: inprogress_assignment_widewide_assigment_fzboost_pz_max_cell_p.hdf5, widewide_assigment_fzboost_pz_max_cell_p
Inserting handle into data store. pz_c_som_pzc_fzboost_pz_max_cell_p: inprogress_pz_c_som_pzc_fzboost_pz_max_cell_p.hdf5, som_pzc_fzboost_pz_max_cell_p
Inserting handle into data store. pc_chat_som_pcchat_fzboost_pz_max_cell_p: inprogress_pc_chat_som_pcchat_fzboost_pz_max_cell_p.hdf5, som_pcchat_fzboost_pz_max_cell_p
Inserting handle into data store. assignment_widespec_assigment_fzboost_pz_max_cell_p: inprogress_assignment_widespec_assigment_fzboost_pz_max_cell_p.hdf5, widespec_assigment_fzboost_pz_max_cell_p
Inserting handle into data store. tomo_bins_wide_som_tomobin_fzboost_pz_max_cell_p: inprogress_tomo_bins_wide_som_tomobin_fzboost_pz_max_cell_p.hdf5, som_tomobin_fzboost_pz_max_cell_p
Inserting handle into data store. output_pz_deepdeep_estimator_knn: inprogress_output_pz_deepdeep_estimator_knn.hdf5, pz_deepdeep_estimator_knn
Inserting handle into data store. output_pz_deepspec_estimator_knn: inprogress_output_pz_deepspec_estimator_knn.hdf5, pz_deepspec_estimator_knn
Inserting handle into data store. output_pz_deepwide_estimator_knn: inprogress_output_pz_deepwide_estimator_knn.hdf5, pz_deepwide_estimator_knn
Inserting handle into data store. output_pz_widewide_estimator_knn: inprogress_output_pz_widewide_estimator_knn.hdf5, pz_widewide_estimator_knn
Inserting handle into data store. output_pz_widespec_estimator_knn: inprogress_output_pz_widespec_estimator_knn.hdf5, pz_widespec_estimator_knn
Inserting handle into data store. assignment_deepspec_assigment_knn_pz_mode: inprogress_assignment_deepspec_assigment_knn_pz_mode.hdf5, deepspec_assigment_knn_pz_mode
Inserting handle into data store. assignment_deepdeep_assigment_knn_pz_mode: inprogress_assignment_deepdeep_assigment_knn_pz_mode.hdf5, deepdeep_assigment_knn_pz_mode
Inserting handle into data store. assignment_deepwide_assigment_knn_pz_mode: inprogress_assignment_deepwide_assigment_knn_pz_mode.hdf5, deepwide_assigment_knn_pz_mode
Inserting handle into data store. assignment_widewide_assigment_knn_pz_mode: inprogress_assignment_widewide_assigment_knn_pz_mode.hdf5, widewide_assigment_knn_pz_mode
Inserting handle into data store. pz_c_som_pzc_knn_pz_mode: inprogress_pz_c_som_pzc_knn_pz_mode.hdf5, som_pzc_knn_pz_mode
Inserting handle into data store. pc_chat_som_pcchat_knn_pz_mode: inprogress_pc_chat_som_pcchat_knn_pz_mode.hdf5, som_pcchat_knn_pz_mode
Inserting handle into data store. assignment_widespec_assigment_knn_pz_mode: inprogress_assignment_widespec_assigment_knn_pz_mode.hdf5, widespec_assigment_knn_pz_mode
Inserting handle into data store. tomo_bins_wide_som_tomobin_knn_pz_mode: inprogress_tomo_bins_wide_som_tomobin_knn_pz_mode.hdf5, som_tomobin_knn_pz_mode
Inserting handle into data store. assignment_deepspec_assigment_knn_pz_max_cell_p: inprogress_assignment_deepspec_assigment_knn_pz_max_cell_p.hdf5, deepspec_assigment_knn_pz_max_cell_p
Inserting handle into data store. assignment_deepdeep_assigment_knn_pz_max_cell_p: inprogress_assignment_deepdeep_assigment_knn_pz_max_cell_p.hdf5, deepdeep_assigment_knn_pz_max_cell_p
Inserting handle into data store. assignment_deepwide_assigment_knn_pz_max_cell_p: inprogress_assignment_deepwide_assigment_knn_pz_max_cell_p.hdf5, deepwide_assigment_knn_pz_max_cell_p
Inserting handle into data store. assignment_widewide_assigment_knn_pz_max_cell_p: inprogress_assignment_widewide_assigment_knn_pz_max_cell_p.hdf5, widewide_assigment_knn_pz_max_cell_p
Inserting handle into data store. pz_c_som_pzc_knn_pz_max_cell_p: inprogress_pz_c_som_pzc_knn_pz_max_cell_p.hdf5, som_pzc_knn_pz_max_cell_p
Inserting handle into data store. pc_chat_som_pcchat_knn_pz_max_cell_p: inprogress_pc_chat_som_pcchat_knn_pz_max_cell_p.hdf5, som_pcchat_knn_pz_max_cell_p
Inserting handle into data store. assignment_widespec_assigment_knn_pz_max_cell_p: inprogress_assignment_widespec_assigment_knn_pz_max_cell_p.hdf5, widespec_assigment_knn_pz_max_cell_p
Inserting handle into data store. tomo_bins_wide_som_tomobin_knn_pz_max_cell_p: inprogress_tomo_bins_wide_som_tomobin_knn_pz_max_cell_p.hdf5, som_tomobin_knn_pz_max_cell_p
Inserting handle into data store. output_pz_deepdeep_estimator_gpz: inprogress_output_pz_deepdeep_estimator_gpz.hdf5, pz_deepdeep_estimator_gpz
Inserting handle into data store. output_pz_deepspec_estimator_gpz: inprogress_output_pz_deepspec_estimator_gpz.hdf5, pz_deepspec_estimator_gpz
Inserting handle into data store. output_pz_deepwide_estimator_gpz: inprogress_output_pz_deepwide_estimator_gpz.hdf5, pz_deepwide_estimator_gpz
Inserting handle into data store. output_pz_widewide_estimator_gpz: inprogress_output_pz_widewide_estimator_gpz.hdf5, pz_widewide_estimator_gpz
Inserting handle into data store. output_pz_widespec_estimator_gpz: inprogress_output_pz_widespec_estimator_gpz.hdf5, pz_widespec_estimator_gpz
Inserting handle into data store. assignment_deepspec_assigment_gpz_pz_mode: inprogress_assignment_deepspec_assigment_gpz_pz_mode.hdf5, deepspec_assigment_gpz_pz_mode
Inserting handle into data store. assignment_deepdeep_assigment_gpz_pz_mode: inprogress_assignment_deepdeep_assigment_gpz_pz_mode.hdf5, deepdeep_assigment_gpz_pz_mode
Inserting handle into data store. assignment_deepwide_assigment_gpz_pz_mode: inprogress_assignment_deepwide_assigment_gpz_pz_mode.hdf5, deepwide_assigment_gpz_pz_mode
Inserting handle into data store. assignment_widewide_assigment_gpz_pz_mode: inprogress_assignment_widewide_assigment_gpz_pz_mode.hdf5, widewide_assigment_gpz_pz_mode
Inserting handle into data store. pz_c_som_pzc_gpz_pz_mode: inprogress_pz_c_som_pzc_gpz_pz_mode.hdf5, som_pzc_gpz_pz_mode
Inserting handle into data store. pc_chat_som_pcchat_gpz_pz_mode: inprogress_pc_chat_som_pcchat_gpz_pz_mode.hdf5, som_pcchat_gpz_pz_mode
Inserting handle into data store. assignment_widespec_assigment_gpz_pz_mode: inprogress_assignment_widespec_assigment_gpz_pz_mode.hdf5, widespec_assigment_gpz_pz_mode
Inserting handle into data store. tomo_bins_wide_som_tomobin_gpz_pz_mode: inprogress_tomo_bins_wide_som_tomobin_gpz_pz_mode.hdf5, som_tomobin_gpz_pz_mode
Inserting handle into data store. assignment_deepspec_assigment_gpz_pz_max_cell_p: inprogress_assignment_deepspec_assigment_gpz_pz_max_cell_p.hdf5, deepspec_assigment_gpz_pz_max_cell_p
Inserting handle into data store. assignment_deepdeep_assigment_gpz_pz_max_cell_p: inprogress_assignment_deepdeep_assigment_gpz_pz_max_cell_p.hdf5, deepdeep_assigment_gpz_pz_max_cell_p
Inserting handle into data store. assignment_deepwide_assigment_gpz_pz_max_cell_p: inprogress_assignment_deepwide_assigment_gpz_pz_max_cell_p.hdf5, deepwide_assigment_gpz_pz_max_cell_p
Inserting handle into data store. assignment_widewide_assigment_gpz_pz_max_cell_p: inprogress_assignment_widewide_assigment_gpz_pz_max_cell_p.hdf5, widewide_assigment_gpz_pz_max_cell_p
Inserting handle into data store. pz_c_som_pzc_gpz_pz_max_cell_p: inprogress_pz_c_som_pzc_gpz_pz_max_cell_p.hdf5, som_pzc_gpz_pz_max_cell_p
Inserting handle into data store. pc_chat_som_pcchat_gpz_pz_max_cell_p: inprogress_pc_chat_som_pcchat_gpz_pz_max_cell_p.hdf5, som_pcchat_gpz_pz_max_cell_p
Inserting handle into data store. assignment_widespec_assigment_gpz_pz_max_cell_p: inprogress_assignment_widespec_assigment_gpz_pz_max_cell_p.hdf5, widespec_assigment_gpz_pz_max_cell_p
Inserting handle into data store. tomo_bins_wide_som_tomobin_gpz_pz_max_cell_p: inprogress_tomo_bins_wide_som_tomobin_gpz_pz_max_cell_p.hdf5, som_tomobin_gpz_pz_max_cell_p
Writing tests/temp_data/projects/ci_test/pipelines/inform_recalib_baseline.yaml
Writing tests/temp_data/projects/ci_test/pipelines/estimate_recalib_baseline.yaml
Writing tests/temp_data/projects/ci_test/pipelines/tomography_baseline.yaml
Inserting handle into data store. output_classify_trainz_equal_count: inprogress_output_classify_trainz_equal_count.hdf5, classify_trainz_equal_count
Inserting handle into data store. tomography_bins: None, true_nz_trainz_equal_count_bin0
Inserting handle into data store. output_classify_trainz_uniform_binning: inprogress_output_classify_trainz_uniform_binning.hdf5, classify_trainz_uniform_binning
Inserting handle into data store. output_classify_simplenn_equal_count: inprogress_output_classify_simplenn_equal_count.hdf5, classify_simplenn_equal_count
Inserting handle into data store. output_classify_simplenn_uniform_binning: inprogress_output_classify_simplenn_uniform_binning.hdf5, classify_simplenn_uniform_binning
Inserting handle into data store. output_classify_fzboost_equal_count: inprogress_output_classify_fzboost_equal_count.hdf5, classify_fzboost_equal_count
Inserting handle into data store. output_classify_fzboost_uniform_binning: inprogress_output_classify_fzboost_uniform_binning.hdf5, classify_fzboost_uniform_binning
Inserting handle into data store. output_classify_knn_equal_count: inprogress_output_classify_knn_equal_count.hdf5, classify_knn_equal_count
Inserting handle into data store. output_classify_knn_uniform_binning: inprogress_output_classify_knn_uniform_binning.hdf5, classify_knn_uniform_binning
Inserting handle into data store. output_classify_gpz_equal_count: inprogress_output_classify_gpz_equal_count.hdf5, classify_gpz_equal_count
Inserting handle into data store. output_classify_gpz_uniform_binning: inprogress_output_classify_gpz_uniform_binning.hdf5, classify_gpz_uniform_binning
[6]:
0
Generate the commands to run the “pz” pipeline in the “baseline” flavor for the “gold” selection
[7]:
single_ceci_command = project.make_pipeline_single_input_command(
pipeline_name="pz",
flavor="basline",
selection="gold",
)
print(single_ceci_command)
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
File ~/checkouts/readthedocs.org/user_builds/rail-projects/envs/latest/lib/python3.11/site-packages/rail/projects/project.py:791, in RailProject.get_flavor(self, name)
790 try:
--> 791 return flavors[name]
792 except KeyError as missing_key:
KeyError: 'basline'
The above exception was the direct cause of the following exception:
KeyError Traceback (most recent call last)
Cell In[7], line 1
----> 1 single_ceci_command = project.make_pipeline_single_input_command(
2 pipeline_name="pz",
3 flavor="basline",
4 selection="gold",
5 )
6 print(single_ceci_command)
File ~/checkouts/readthedocs.org/user_builds/rail-projects/envs/latest/lib/python3.11/site-packages/rail/projects/project.py:592, in RailProject.make_pipeline_single_input_command(self, pipeline_name, flavor, **kwargs)
590 pipeline_template = self.get_pipeline(pipeline_name)
591 pipeline_instance = pipeline_template.make_instance(self, flavor, {})
--> 592 return pipeline_instance.make_pipeline_single_input_command(self, **kwargs)
File ~/checkouts/readthedocs.org/user_builds/rail-projects/envs/latest/lib/python3.11/site-packages/rail/projects/pipeline_holder.py:906, in RailPipelineInstance.make_pipeline_single_input_command(self, project, **kwargs)
904 pipeline_path = self.config.path
905 pipeline_config = pipeline_path.replace(".yaml", "_config.yml")
--> 906 input_files = self.get_input_files(project, **kwargs)
907 sink_dir = input_files.pop("sink_dir")
908 command_line = project.generate_ceci_command(
909 pipeline_path=pipeline_path,
910 config=pipeline_config,
(...) 913 log_dir=f"{sink_dir}/logs",
914 )
File ~/checkouts/readthedocs.org/user_builds/rail-projects/envs/latest/lib/python3.11/site-packages/rail/projects/pipeline_holder.py:878, in RailPipelineInstance.get_input_files(self, project, **kwargs)
874 sink_dir = project.get_path(
875 "ceci_output_dir", flavor=self.config.flavor, **kwargs
876 )
877 input_callback = INPUT_CALLBACK_DICT[pipeline_name]
--> 878 input_files = input_callback(
879 project, pipeline_name, sink_dir, flavor=self.config.flavor, **kwargs
880 )
881 input_files.setdefault("sink_dir", sink_dir)
882 return input_files
File ~/checkouts/readthedocs.org/user_builds/rail-projects/envs/latest/lib/python3.11/site-packages/rail/projects/pipeline_holder.py:489, in pz_input_callback(project, pipeline_name, sink_dir, **kwargs)
487 if input_file_flavor!=flavor:
488 input_file_flavor=flavor
--> 489 input_files[key] = project.get_file_for_flavor(
490 input_file_flavor, val["tag"], **kwargs
491 )
492 return input_files
File ~/checkouts/readthedocs.org/user_builds/rail-projects/envs/latest/lib/python3.11/site-packages/rail/projects/project.py:802, in RailProject.get_file_for_flavor(self, flavor, label, **kwargs)
797 def get_file_for_flavor(self, flavor: str, label: str, **kwargs: Any) -> str:
798 """Resolve the file associated to a particular flavor and label
799
800 E.g., flavor=baseline and label=train would give the baseline training file
801 """
--> 802 flavor_dict = self.get_flavor(flavor)
803 try:
804 file_alias = flavor_dict.config.file_aliases[label]
File ~/checkouts/readthedocs.org/user_builds/rail-projects/envs/latest/lib/python3.11/site-packages/rail/projects/project.py:793, in RailProject.get_flavor(self, name)
791 return flavors[name]
792 except KeyError as missing_key:
--> 793 raise KeyError(
794 f"flavor '{name}' not found in {list(flavors.keys())}"
795 ) from missing_key
KeyError: "flavor 'basline' not found in ['baseline', 'train_cosmos', 'gpz_gl']"
Generate the commands to run the “spec_selection” pipeline in the “baseline” flavor for the “gold” selection
[8]:
ceci_catalog_commands = project.make_pipeline_catalog_commands(
pipeline_name="spec_selection",
flavor="basline",
selection="gold",
spec_selections = list(project.get_spec_selections().keys()),
)
print(ceci_catalog_commands)
[([['mkdir', '-p', 'tests/temp_data/data/ci_test_v1.1.3_gold_basline/9924'], ['ceci', 'tests/temp_data/projects/ci_test/pipelines/spec_selection_basline.yaml', 'config=tests/temp_data/projects/ci_test/pipelines/spec_selection_basline_config.yml', 'output_dir=tests/temp_data/data/ci_test_v1.1.3_gold_basline/9924', 'log_dir=tests/temp_data/data/ci_test_v1.1.3_gold_basline/9924', 'inputs.input=tests/temp_data/data/ci_test_v1.1.3_gold_basline/9924/output_dereddener_errors.pq'], ['tables-io', 'convert', '--input', 'tests/temp_data/data/ci_test_v1.1.3_gold_basline/9924/output_select_zCOSMOS.pq', '--output', 'tests/temp_data/data/ci_test_v1.1.3_gold_basline/9924/output_select_zCOSMOS.hdf5']], 'tests/temp_data/data/ci_test_v1.1.3_gold_basline/9924/submit_spec_selection_gold_basline.sh'), ([['mkdir', '-p', 'tests/temp_data/data/ci_test_v1.1.3_gold_basline/9925'], ['ceci', 'tests/temp_data/projects/ci_test/pipelines/spec_selection_basline.yaml', 'config=tests/temp_data/projects/ci_test/pipelines/spec_selection_basline_config.yml', 'output_dir=tests/temp_data/data/ci_test_v1.1.3_gold_basline/9925', 'log_dir=tests/temp_data/data/ci_test_v1.1.3_gold_basline/9925', 'inputs.input=tests/temp_data/data/ci_test_v1.1.3_gold_basline/9925/output_dereddener_errors.pq'], ['tables-io', 'convert', '--input', 'tests/temp_data/data/ci_test_v1.1.3_gold_basline/9925/output_select_zCOSMOS.pq', '--output', 'tests/temp_data/data/ci_test_v1.1.3_gold_basline/9925/output_select_zCOSMOS.hdf5']], 'tests/temp_data/data/ci_test_v1.1.3_gold_basline/9925/submit_spec_selection_gold_basline.sh')]
[ ]:
[ ]: