pytabkit.bench package

Subpackages

• pytabkit.bench.alg_wrappers package
  • Submodules
  • pytabkit.bench.alg_wrappers.general module
    • AlgWrapper
      • AlgWrapper.__init__()
      • AlgWrapper.get_max_n_vectorized()
      • AlgWrapper.get_pred_param_names()
      • AlgWrapper.get_required_resources()
      • AlgWrapper.run()
  • pytabkit.bench.alg_wrappers.interface_wrappers module
    • AlgInterfaceWrapper
      • AlgInterfaceWrapper.__init__()
      • AlgInterfaceWrapper.create_alg_interface()
      • AlgInterfaceWrapper.get_pred_param_names()
      • AlgInterfaceWrapper.get_required_resources()
      • AlgInterfaceWrapper.run()
    • AlgorithmSelectionWrapper
      • AlgorithmSelectionWrapper.__init__()
      • AlgorithmSelectionWrapper.get_required_resources()
    • AutoGluonModelInterfaceWrapper
      • AutoGluonModelInterfaceWrapper.__init__()
    • CaruanaEnsembleWrapper
      • CaruanaEnsembleWrapper.__init__()
      • CaruanaEnsembleWrapper.get_required_resources()
    • CatBoostHyperoptInterfaceWrapper
      • CatBoostHyperoptInterfaceWrapper.create_single_alg_interface()
    • CatBoostInterfaceWrapper
      • CatBoostInterfaceWrapper.create_sub_split_interface()
    • CatBoostSklearnInterfaceWrapper
      • CatBoostSklearnInterfaceWrapper.create_sub_split_interface()
    • ExtraTreesInterfaceWrapper
      • ExtraTreesInterfaceWrapper.create_sub_split_interface()
    • FTTransformerInterfaceWrapper
      • FTTransformerInterfaceWrapper.create_sub_split_interface()
    • GBTInterfaceWrapper
      • GBTInterfaceWrapper.create_sub_split_interface()
    • GrandeInterfaceWrapper
      • GrandeInterfaceWrapper.create_sub_split_interface()
    • KANInterfaceWrapper
      • KANInterfaceWrapper.create_sub_split_interface()
    • KNNInterfaceWrapper
      • KNNInterfaceWrapper.create_sub_split_interface()
    • LGBMHyperoptInterfaceWrapper
      • LGBMHyperoptInterfaceWrapper.create_single_alg_interface()
    • LGBMInterfaceWrapper
      • LGBMInterfaceWrapper.create_sub_split_interface()
    • LGBMSklearnInterfaceWrapper
      • LGBMSklearnInterfaceWrapper.create_sub_split_interface()
    • LinearModelInterfaceWrapper
      • LinearModelInterfaceWrapper.create_sub_split_interface()
    • LoadResultsWrapper
      • LoadResultsWrapper.__init__()
      • LoadResultsWrapper.get_required_resources()
    • MLPRTDLInterfaceWrapper
      • MLPRTDLInterfaceWrapper.create_sub_split_interface()
    • MultiSplitAlgInterfaceWrapper
      • MultiSplitAlgInterfaceWrapper.__init__()
      • MultiSplitAlgInterfaceWrapper.create_single_alg_interface()
    • NNHyperoptInterfaceWrapper
      • NNHyperoptInterfaceWrapper.__init__()
      • NNHyperoptInterfaceWrapper.get_max_n_vectorized()
    • NNInterfaceWrapper
      • NNInterfaceWrapper.__init__()
      • NNInterfaceWrapper.get_max_n_vectorized()
    • RFInterfaceWrapper
      • RFInterfaceWrapper.create_sub_split_interface()
    • RandomParamsCatBoostInterfaceWrapper
      • RandomParamsCatBoostInterfaceWrapper.create_single_alg_interface()
    • RandomParamsExtraTreesInterfaceWrapper
      • RandomParamsExtraTreesInterfaceWrapper.__init__()
    • RandomParamsFTTransformerInterfaceWrapper
      • RandomParamsFTTransformerInterfaceWrapper.__init__()
    • RandomParamsKNNInterfaceWrapper
      • RandomParamsKNNInterfaceWrapper.__init__()
    • RandomParamsLGBMInterfaceWrapper
      • RandomParamsLGBMInterfaceWrapper.create_single_alg_interface()
    • RandomParamsLinearModelInterfaceWrapper
      • RandomParamsLinearModelInterfaceWrapper.__init__()
    • RandomParamsNNInterfaceWrapper
      • RandomParamsNNInterfaceWrapper.__init__()
    • RandomParamsRFInterfaceWrapper
      • RandomParamsRFInterfaceWrapper.__init__()
    • RandomParamsRTDLMLPInterfaceWrapper
      • RandomParamsRTDLMLPInterfaceWrapper.__init__()
    • RandomParamsResnetInterfaceWrapper
      • RandomParamsResnetInterfaceWrapper.__init__()
    • RandomParamsTabRInterfaceWrapper
      • RandomParamsTabRInterfaceWrapper.create_single_alg_interface()
    • RandomParamsXGBInterfaceWrapper
      • RandomParamsXGBInterfaceWrapper.create_single_alg_interface()
    • RandomParamsxRFMInterfaceWrapper
      • RandomParamsxRFMInterfaceWrapper.create_single_alg_interface()
    • ResNetRTDLInterfaceWrapper
      • ResNetRTDLInterfaceWrapper.create_sub_split_interface()
    • SklearnMLPInterfaceWrapper
      • SklearnMLPInterfaceWrapper.create_sub_split_interface()
    • SubSplitInterfaceWrapper
      • SubSplitInterfaceWrapper.__init__()
      • SubSplitInterfaceWrapper.create_single_alg_interface()
      • SubSplitInterfaceWrapper.create_sub_split_interface()
    • TabICLInterfaceWrapper
      • TabICLInterfaceWrapper.create_sub_split_interface()
    • TabMInterfaceWrapper
      • TabMInterfaceWrapper.create_sub_split_interface()
    • TabPFN2InterfaceWrapper
      • TabPFN2InterfaceWrapper.create_sub_split_interface()
    • TabRInterfaceWrapper
      • TabRInterfaceWrapper.create_sub_split_interface()
    • XGBHyperoptInterfaceWrapper
      • XGBHyperoptInterfaceWrapper.create_single_alg_interface()
    • XGBInterfaceWrapper
      • XGBInterfaceWrapper.create_sub_split_interface()
    • XGBSklearnInterfaceWrapper
      • XGBSklearnInterfaceWrapper.create_sub_split_interface()
    • get_prep_factory()
    • xRFMInterfaceWrapper
      • xRFMInterfaceWrapper.create_sub_split_interface()
  • Module contents
• pytabkit.bench.data package
  • Submodules
  • pytabkit.bench.data.common module
    • SplitType
      • SplitType.DEFAULT
      • SplitType.RANDOM
    • TaskSource
      • TaskSource.AUTOML_CLASS_SMALL
      • TaskSource.CUSTOM
      • TaskSource.OPENML_CLASS
      • TaskSource.OPENML_CLASS_BIN_EXTRA
      • TaskSource.OPENML_REGRESSION
      • TaskSource.TABARENA_CLASS
      • TaskSource.TABARENA_REG
      • TaskSource.UCI_BIN_CLASS
      • TaskSource.UCI_MULTI_CLASS
      • TaskSource.UCI_REGRESSION
  • pytabkit.bench.data.get_uci module
    • download_all_uci()
    • get_EEG_steady_state()
    • get_KDC_4007()
    • get_abalone()
    • get_activity_recognition()
    • get_air_quality()
    • get_anuran_calls()
    • get_appliances_energy()
    • get_arabic_digit()
    • get_artificial_characters()
    • get_assamese_characters()
    • get_australian_sign_language()
    • get_avila()
    • get_bach_chorals_harmony()
    • get_bank_marketing()
    • get_bejing_pm25()
    • get_bike_sharing()
    • get_bitcoin_heist()
    • get_ble_rssi_indoor_location()
    • get_blood_pressure()
    • get_carbon_nanotubes()
    • get_cargo_2000()
    • get_census_income()
    • get_character_trajectories()
    • get_chess()
    • get_chess_krvk()
    • get_crop_mapping()
    • get_crowd_sourced_mapping()
    • get_cycle_power_plant()
    • get_default_credit_card()
    • get_eeg_eye_state()
    • get_eeg_steady_state_visual()
    • get_electrical_grid_stability_simulated()
    • get_emg_for_gestures()
    • get_emg_physical_action()
    • get_epileptic_seizure_recognition()
    • get_facebook_comment_volume()
    • get_facebook_live_sellers_thailand()
    • get_firewall()
    • get_firm_teacher_clave()
    • get_first_order_theorem_proving()
    • get_five_cities_pm25()
    • get_gas_sensor_drift()
    • get_gas_turbine()
    • get_gesture_phase_segmentation()
    • get_gnfuv_unmanned_surface_vehicles()
    • get_grammatical_facial_expressions()
    • get_hiv_1_protease()
    • get_htru2()
    • get_human_activity_smartphone()
    • get_indoor_channel_measurements()
    • get_indoor_loc()
    • get_indoor_loc_mag()
    • get_indoor_user_movement_prediction()
    • get_insurance_benchmark()
    • get_isolet()
    • get_landsat_satimage()
    • get_letter_recognition()
    • get_madelon()
    • get_magic_gamma_telescope()
    • get_metro_interstate_traffic_volume()
    • get_meu_mobile_ksd()
    • get_mushroom()
    • get_musk()
    • get_naval_propulsion()
    • get_nomao()
    • get_nursery()
    • get_occupancy_detection()
    • get_online_news_popularity()
    • get_online_shoppers_attention()
    • get_opportunity_activity()
    • get_optical_recognition_handwritten_digits()
    • get_oral_toxicity()
    • get_ozone_level()
    • get_page_blocks()
    • get_parking_birmingham()
    • get_parkinson()
    • get_pen_recognition_handwritten_characters()
    • get_phishing()
    • get_pmu_ud()
    • get_polish_companies_bankruptcy()
    • get_protein_tertiary_structure()
    • get_query_analytics()
    • get_real_estate_value()
    • get_seismic_bumps()
    • get_seoul_bike_data()
    • get_shill_bidding()
    • get_simulated_falls()
    • get_skill_craft()
    • get_smartphone_human_activity()
    • get_smartphone_human_activity_postural()
    • get_sml2010()
    • get_south_german_credit()
    • get_spambase()
    • get_superconductivity()
    • get_tamilnadu_electricity()
    • get_tarvel_review_ratings()
    • get_thyroids()
    • get_turkiye_student_evaluation()
    • get_vicon_physical_action()
    • get_wall_following_robot()
    • get_wave_energy()
    • get_waveform()
    • get_wilt()
    • get_wine_quality()
  • pytabkit.bench.data.import_talent_benchmark module
    • import_talent_benchmark()
  • pytabkit.bench.data.import_tasks module
    • PandasTask
      • PandasTask.__init__()
      • PandasTask.deduplicate()
      • PandasTask.from_openml_task_id()
      • PandasTask.get_n_classes()
      • PandasTask.get_n_samples()
      • PandasTask.get_task()
      • PandasTask.limit_n_classes()
      • PandasTask.normalize_regression_y()
      • PandasTask.remove_missing_cont()
      • PandasTask.subsample()
    • check_zero_hot()
    • convert_to_class_numbers()
    • download_if_not_exists()
    • extract_categories()
    • get_openml_ds_names()
    • get_openml_task_ids()
    • import_from_csv()
    • import_openml()
    • import_uci_tasks()
    • set_openml_cache_dir()
  • pytabkit.bench.data.paths module
    • Paths
      • Paths.__init__()
      • Paths.algs()
      • Paths.base()
      • Paths.eval()
      • Paths.from_env_variables()
      • Paths.new_tmp_folder()
      • Paths.plots()
      • Paths.resources()
      • Paths.resources_exp_it()
      • Paths.result_summaries()
      • Paths.results()
      • Paths.results_alg_task()
      • Paths.results_alg_task_split()
      • Paths.results_task()
      • Paths.summary_alg_task()
      • Paths.task_collections()
      • Paths.task_source()
      • Paths.tasks()
      • Paths.tasks_task()
      • Paths.times()
      • Paths.times_alg_task()
      • Paths.tmp()
      • Paths.uci_download()
    • TmpPathContextManager
      • TmpPathContextManager.__init__()
  • pytabkit.bench.data.tasks module
    • Task
      • Task.__init__()
      • Task.save()
    • TaskCollection
      • TaskCollection.__init__()
      • TaskCollection.from_name()
      • TaskCollection.from_source()
      • TaskCollection.load_infos()
      • TaskCollection.save()
    • TaskDescription
      • TaskDescription.__init__()
      • TaskDescription.exists_task()
      • TaskDescription.from_dict()
      • TaskDescription.load_info()
      • TaskDescription.load_task()
      • TaskDescription.to_dict()
    • TaskInfo
      • TaskInfo.__init__()
      • TaskInfo.from_ds()
      • TaskInfo.get_default_splits()
      • TaskInfo.get_ds_size_gb()
      • TaskInfo.get_n_classes()
      • TaskInfo.get_random_splits()
      • TaskInfo.load()
      • TaskInfo.load_task()
      • TaskInfo.save()
    • TaskPackage
      • TaskPackage.__init__()
  • pytabkit.bench.data.uci_file_ops module
    • UCIVars
      • UCIVars.binary_classification_data_folder
      • UCIVars.data_folder
      • UCIVars.data_group_id
      • UCIVars.multiclass_classification_data_folder
      • UCIVars.raw_data_folder
      • UCIVars.regression_data_folder
      • UCIVars.statistics_filename
    • auto_replace_categories_in_mixed_data()
    • auto_replace_missing_in_mixed_data()
    • concat_files()
    • convert_replace_string_to_vector()
    • convert_time_to_seconds()
    • count_bin_columns()
    • download_and_save()
    • get_categories_in_mixed_data()
    • get_category_replace_string()
    • is_number()
    • load_mixed_raw_data()
    • load_raw_data()
    • make_folder()
    • move_label_in_front()
    • my_decode()
    • prepare_new_data_set_group_id()
    • remove_columns()
    • remove_empty_columns()
    • remove_files()
    • remove_rows_with_label()
    • replace_bin_cats_in_mixed_data()
    • replace_categories_in_file()
    • replace_categories_in_mixed_data()
    • replace_chars_in_file()
    • replace_circulars_in_mixed_data()
    • replace_isodate_by_day_in_mixed_data()
    • replace_manual_in_mixed_data()
    • replace_ordinals_in_mixed_data()
    • replace_time_by_seconds_in_mixed_data()
    • save_data_stats()
    • save_data_to_file()
    • un_z_raw_data()
    • unarff_raw_data()
    • ungz_raw_data()
    • unrar_raw_data()
    • untar_raw_data()
    • unzip_raw_data()
    • write_mixed_raw_data()
  • Module contents
• pytabkit.bench.eval package
  • Submodules
  • pytabkit.bench.eval.analysis module
    • ResultsTables
      • ResultsTables.__init__()
      • ResultsTables.get()
    • get_benchmark_results()
    • get_display_name()
    • get_ensemble_groups()
    • get_opt_groups()
    • get_simplified_name()
    • get_t_mean_confidence_interval()
  • pytabkit.bench.eval.colors module
    • bilin_int()
    • bisection_find()
    • more_percep_uniform_hue()
  • pytabkit.bench.eval.evaluation module
    • AlgFilter
    • AlgTaskTable
      • AlgTaskTable.__init__()
      • AlgTaskTable.filter_algs()
      • AlgTaskTable.filter_n_splits()
      • AlgTaskTable.map()
      • AlgTaskTable.rename_algs()
      • AlgTaskTable.to_array()
    • ArrayTableAnalyzer
      • ArrayTableAnalyzer.__init__()
      • ArrayTableAnalyzer.print_analysis()
    • DefaultEvalModeSelector
      • DefaultEvalModeSelector.select_eval_modes()
    • EvalModeSelector
      • EvalModeSelector.select()
      • EvalModeSelector.select_eval_modes()
    • FunctionAlgFilter
      • FunctionAlgFilter.__init__()
    • GreedyAlgSelectionTableAnalyzer
    • MeanTableAnalyzer
      • MeanTableAnalyzer.__init__()
      • MeanTableAnalyzer.get_intervals()
      • MeanTableAnalyzer.get_means()
      • MeanTableAnalyzer.print_analysis()
    • MultiResultsTable
      • MultiResultsTable.__init__()
      • MultiResultsTable.get_test_results_table()
      • MultiResultsTable.load()
    • NormalizedLossTableAnalyzer
    • RankTableAnalyzer
    • TableAnalyzer
      • TableAnalyzer.__init__()
      • TableAnalyzer.print_analysis()
    • TaskWeighting
      • TaskWeighting.__init__()
      • TaskWeighting.get_n_groups()
      • TaskWeighting.get_task_weights()
    • WinsTableAnalyzer
    • alg_comparison_str()
    • alg_results_str()
    • get_ranks()
  • pytabkit.bench.eval.plotting module
    • coll_name_to_title()
    • extend_runtimes()
    • get_equidistant_blue_colors()
    • get_equidistant_colors()
    • get_plot_color()
    • get_plot_color_idx()
    • gg_color_hue()
    • plot_benchmark_bars()
    • plot_cdd()
    • plot_cdd_ax()
    • plot_cumulative_ablations()
    • plot_pareto()
    • plot_pareto_ax()
    • plot_scatter()
    • plot_scatter_ax()
    • plot_schedule()
    • plot_schedules()
    • plot_stopping()
    • plot_stopping_ax()
    • plot_winrates()
    • shorten_coll_names()
  • pytabkit.bench.eval.runtimes module
    • get_avg_predict_times()
    • get_avg_train_times()
  • pytabkit.bench.eval.tables module
    • generate_ablations_table()
    • generate_architecture_table()
    • generate_collections_table()
    • generate_ds_table()
    • generate_individual_results_table()
    • generate_preprocessing_table()
    • generate_refit_table()
    • generate_stopping_table()
  • Module contents
• pytabkit.bench.run package
  • Submodules
  • pytabkit.bench.run.results module
    • ResultManager
      • ResultManager.__init__()
      • ResultManager.add_results()
      • ResultManager.load()
      • ResultManager.save()
    • save_summaries()
  • pytabkit.bench.run.task_execution module
    • RunConfig
      • RunConfig.__init__()
    • TabBenchJob
      • TabBenchJob.__init__()
      • TabBenchJob.get_desc()
      • TabBenchJob.get_group()
      • TabBenchJob.get_required_resources()
    • TabBenchJobManager
      • TabBenchJobManager.__init__()
      • TabBenchJobManager.add_jobs()
      • TabBenchJobManager.run_jobs()
    • run_alg_selection()
  • Module contents
• pytabkit.bench.scheduling package
  • Submodules
  • pytabkit.bench.scheduling.execution module
    • NodeManager
      • NodeManager.start()
      • NodeManager.terminate()
    • RayJobManager
      • RayJobManager.__init__()
      • RayJobManager.get_resource_manager()
      • RayJobManager.pop_finished_job_infos()
      • RayJobManager.start()
      • RayJobManager.submit_job()
      • RayJobManager.terminate()
    • get_gpu_rams_gb()
    • measure_node_resources()
    • node_runner()
  • pytabkit.bench.scheduling.jobs module
    • AbstractJob
      • AbstractJob.get_desc()
      • AbstractJob.get_group()
      • AbstractJob.get_required_resources()
    • JobResult
      • JobResult.__init__()
      • JobResult.set_max_cpu_ram_gb()
    • JobRunner
      • JobRunner.__init__()
  • pytabkit.bench.scheduling.resource_manager module
    • JobInfo
      • JobInfo.__init__()
      • JobInfo.get_status()
      • JobInfo.is_failed()
      • JobInfo.is_finished()
      • JobInfo.is_remaining()
      • JobInfo.is_running()
      • JobInfo.is_succeed()
      • JobInfo.set_finished()
      • JobInfo.set_started()
    • JobStatus
      • JobStatus.FAILED
      • JobStatus.REMAINING
      • JobStatus.RUNNING
      • JobStatus.SUCCEEDED
    • ResourceManager
      • ResourceManager.__init__()
      • ResourceManager.get_fixed_resources()
      • ResourceManager.get_free_resources()
      • ResourceManager.get_total_resources()
      • ResourceManager.job_finished()
      • ResourceManager.job_started()
  • pytabkit.bench.scheduling.resources module
    • NodeResources
      • NodeResources.__init__()
      • NodeResources.get_cpu_ram_gb()
      • NodeResources.get_gpu_rams_gb()
      • NodeResources.get_gpu_usages()
      • NodeResources.get_interface_resources()
      • NodeResources.get_n_physical_cores()
      • NodeResources.get_n_threads()
      • NodeResources.get_physical_core_usages()
      • NodeResources.get_resource_vector()
      • NodeResources.get_total_gpu_ram_gb()
      • NodeResources.get_total_gpu_usage()
      • NodeResources.get_used_gpu_ids()
      • NodeResources.get_used_physical_cores()
      • NodeResources.set_cpu_ram_gb()
      • NodeResources.set_gpu_rams_gb()
      • NodeResources.set_n_threads()
      • NodeResources.try_assign()
      • NodeResources.zeros_like()
    • SystemResources
      • SystemResources.__init__()
      • SystemResources.get_cpu_ram_gb()
      • SystemResources.get_gpu_ram_gb()
      • SystemResources.get_gpu_usage()
      • SystemResources.get_n_threads()
      • SystemResources.get_num_gpus()
      • SystemResources.get_resource_vector()
  • pytabkit.bench.scheduling.schedulers module
    • BaseJobScheduler
      • BaseJobScheduler.__init__()
      • BaseJobScheduler.add_jobs()
      • BaseJobScheduler.run()
    • CustomJobScheduler
    • SimpleJobScheduler
    • format_date_s()
    • format_length_s()
  • Module contents

Module contents