Index

Symbols

**kwargs (fts_examples.profiling.memprofiler.MemProfiler parameter), [1]

*args (fts_examples.profiling.memprofiler.MemProfiler parameter), [1]

_

_fts_state (finetuning_scheduler.fts.FinetuningScheduler attribute)

_state (fts_examples.profiling.memprofiler.MemProfiler attribute)

A

add_memprofiler_hooks() (fts_examples.profiling.memprofiler.MemProfiler method) add_optimizer_groups() (finetuning_scheduler.fts_supporters.ScheduleImplMixin static method) agg_func (finetuning_scheduler.fts_supporters.ScheduleImplMixin.sync parameter) allow_untested (finetuning_scheduler.fts.FinetuningScheduler parameter), [1]

apply_lambdas (finetuning_scheduler.fts_supporters.ScheduleImplMixin.add_optimizer_groups parameter) apply_lambdas_new_pgs (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] asets (finetuning_scheduler.fts_supporters.ScheduleImplMixin.sync parameter) awp_overrides (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter attribute) (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter parameter)

B

base_ft_phase() (finetuning_scheduler.strategy_adapters.StrategyAdapter static method) base_max_lr (finetuning_scheduler.fts.FinetuningScheduler attribute) (finetuning_scheduler.fts.FinetuningScheduler parameter), [1]

best_ckpt_depth (finetuning_scheduler.fts_supporters.FTSCheckpoint attribute)

C

callback_attrs (finetuning_scheduler.fts_supporters.CallbackResolverMixin parameter), [1] callback_dep_parents (finetuning_scheduler.fts_supporters.CallbackDepMixin parameter), [1] callback_parents (finetuning_scheduler.fts_supporters.CallbackResolverMixin parameter), [1] CallbackDepMixin (class in finetuning_scheduler.fts_supporters) CallbackResolverMixin (class in finetuning_scheduler.fts_supporters) check_on_train_epoch_end (finetuning_scheduler.fts_supporters.FTSEarlyStopping attribute) checkpoint_connector (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter.load_optimizer_state_dict parameter) collect_funcs (fts_examples.profiling.config.MemProfilerCfg parameter), [1]

connect() (finetuning_scheduler.strategy_adapters.StrategyAdapter method) (fts_examples.profiling.memprofiler.MemProfiler method) connect_callback() (finetuning_scheduler.fts_supporters.CallbackResolverMixin method) construct_mapping() (finetuning_scheduler.fts_supporters.UniqueKeyLoader method) cuda_allocator_history (fts_examples.profiling.config.MemProfilerCfg parameter), [1] cuda_allocator_history_snap() (fts_examples.profiling.memprofiler.MemProfiler method) curr_depth (finetuning_scheduler.fts.FinetuningScheduler property) current_ckpt_depth (finetuning_scheduler.fts_supporters.FTSCheckpoint attribute) custom_strategy_adapter (finetuning_scheduler.fts.FinetuningScheduler parameter), [1]

D

depth (finetuning_scheduler.fts.FinetuningScheduler.step_pg parameter) (finetuning_scheduler.fts_supporters.ScheduleImplMixin.thaw_to_depth parameter) depth_remaining (finetuning_scheduler.fts.FinetuningScheduler property) depth_sync (finetuning_scheduler.fts.FinetuningScheduler.step_pg parameter) done() (fts_examples.profiling.memprofiler.MemProfiler method)

dump_loc (finetuning_scheduler.fts_supporters.ScheduleImplMixin.gen_ft_schedule parameter) (finetuning_scheduler.fts_supporters.ScheduleImplMixin.save_schedule parameter) dump_memory_stats() (fts_examples.profiling.memprofiler.MemProfiler method) dump_memorystats_pickle (fts_examples.profiling.config.MemProfilerCfg parameter), [1] dump_memorystats_yaml (fts_examples.profiling.config.MemProfilerCfg parameter), [1]

E

enable_memory_hooks (fts_examples.profiling.config.MemProfilerCfg parameter), [1] enable_saved_tensors_hooks (fts_examples.profiling.config.MemProfilerCfg parameter), [1] enabled (fts_examples.profiling.config.MemProfilerCfg parameter), [1] enforce_phase0_params (finetuning_scheduler.fts.FinetuningScheduler parameter), [1]

epoch_transitions_only (finetuning_scheduler.fts.FinetuningScheduler attribute) (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] es_phase_complete (finetuning_scheduler.fts_supporters.FTSEarlyStopping attribute) exec_reset_state_hooks() (fts_examples.profiling.memprofiler.MemProfiler method)

F

final_phase (finetuning_scheduler.fts_supporters.FTSEarlyStopping attribute) finetuning_scheduler.dynamic_versioning module finetuning_scheduler.fts module finetuning_scheduler.fts_supporters module finetuning_scheduler.strategy_adapters module FinetuningScheduler (class in finetuning_scheduler.fts) finetuningscheduler_callback (finetuning_scheduler.fts_supporters.FTSCheckpoint attribute) (finetuning_scheduler.fts_supporters.FTSEarlyStopping attribute) fn_name (fts_examples.profiling.memprofiler.MemProfiler.gen_snap_keys parameter) (fts_examples.profiling.memprofiler.MemProfiler.memprofile_ctx parameter) (fts_examples.profiling.memprofiler.MemProfiler.memprofile_fsdp_ctx parameter) (fts_examples.profiling.memprofiler.MemProfiler.memprofile_meta_ctx parameter) (fts_examples.profiling.memprofiler.MemProfiler.memprofile_snap_ctx parameter) (fts_examples.profiling.memprofiler.MemProfiler.snap parameter) (fts_examples.profiling.memprofiler.MemProfiler.teardown_prof parameter) (fts_examples.profiling.memprofiler.MemProfiler.update_collect_state parameter) freeze_before_training() (finetuning_scheduler.fts.FinetuningScheduler method) frozen_bn_track_running_stats (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] fsdp_default_kwargs (finetuning_scheduler.strategy_adapters.ModelParallelStrategyAdapter attribute) (finetuning_scheduler.strategy_adapters.ModelParallelStrategyAdapter parameter)

fsdp_mem_track_module_depth (fts_examples.profiling.config.MemProfilerCfg parameter), [1] fsdp_mem_tracker (fts_examples.profiling.memprofiler.MemProfiler attribute) fsdp_mem_tracker_root_module (fts_examples.profiling.config.MemProfilerCfg parameter), [1] (fts_examples.profiling.memprofiler.MemProfiler property) fsdp_mem_tracker_tabulate (fts_examples.profiling.config.MemProfilerCfg parameter), [1] fsdp_mem_tracker_units (fts_examples.profiling.config.MemProfilerCfg parameter), [1] fsdp_memory_stats (fts_examples.profiling.memprofiler.MemProfiler attribute) fsdp_param_transform() (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter method) fsdp_plan (finetuning_scheduler.strategy_adapters.ModelParallelStrategyAdapter attribute) (finetuning_scheduler.strategy_adapters.ModelParallelStrategyAdapter parameter) FSDPStrategyAdapter (class in finetuning_scheduler.strategy_adapters) ft_schedule (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] fts_examples.profiling.config module fts_examples.profiling.memprofiler module fts_optim_transform() (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter method) (finetuning_scheduler.strategy_adapters.StrategyAdapter method) fts_parent (finetuning_scheduler.strategy_adapters.StrategyAdapter.connect parameter) FTSCheckpoint (class in finetuning_scheduler.fts_supporters) FTSEarlyStopping (class in finetuning_scheduler.fts_supporters) FTSState (class in finetuning_scheduler.fts_supporters) func (fts_examples.profiling.memprofiler.MemProfiler.memprofilable parameter)

G

gen_ft_sched_only (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] gen_ft_schedule() (finetuning_scheduler.fts_supporters.ScheduleImplMixin static method)

gen_implicit_schedule() (finetuning_scheduler.fts_supporters.ScheduleImplMixin method) gen_or_load_sched() (finetuning_scheduler.fts_supporters.ScheduleImplMixin method) gen_snap_keys() (fts_examples.profiling.memprofiler.MemProfiler method)

I

init_ft_sched() (finetuning_scheduler.fts_supporters.ScheduleImplMixin method) init_fts() (finetuning_scheduler.fts_supporters.ScheduleImplMixin method) init_memprof_log_dir() (fts_examples.profiling.memprofiler.MemProfiler method) init_params (finetuning_scheduler.fts_supporters.ScheduleParsingMixin.reinit_optimizer parameter) init_thaw (finetuning_scheduler.strategy_adapters.StrategyAdapter.base_ft_phase parameter) inspect_only (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter.fsdp_param_transform parameter) (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter.fts_optim_transform parameter) (finetuning_scheduler.strategy_adapters.StrategyAdapter.fts_optim_transform parameter)

iter_ctx (fts_examples.profiling.memprofiler.MemProfiler.gen_snap_keys parameter) (fts_examples.profiling.memprofiler.MemProfiler.snap parameter) (fts_examples.profiling.memprofiler.MemProfiler.teardown_prof parameter) (fts_examples.profiling.memprofiler.MemProfiler.update_collect_state parameter) iter_idx (fts_examples.profiling.memprofiler.MemProfiler.done parameter) (fts_examples.profiling.memprofiler.MemProfiler.gen_snap_keys parameter) (fts_examples.profiling.memprofiler.MemProfiler.update_collect_state parameter)

L

layer_config (finetuning_scheduler.fts_supporters.ScheduleImplMixin.save_schedule parameter) lightning_restore_optimizer (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter property) load_optimizer_state_dict() (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter method) load_state_dict() (finetuning_scheduler.fts.FinetuningScheduler method) (finetuning_scheduler.fts_supporters.FTSCheckpoint method) load_yaml_schedule() (finetuning_scheduler.fts_supporters.ScheduleImplMixin static method)

log_dir (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] (finetuning_scheduler.fts.FinetuningScheduler property) logging_level (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] logical_param_translation() (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter method) (finetuning_scheduler.strategy_adapters.StrategyAdapter method) lr (finetuning_scheduler.fts_supporters.ScheduleImplMixin.add_optimizer_groups parameter)

M

max_depth (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] maybe_init_fsdp_mem_tracker() (fts_examples.profiling.memprofiler.MemProfiler method) memory_hooks (fts_examples.profiling.config.MemProfilerCfg parameter), [1] memory_stats (fts_examples.profiling.memprofiler.MemProfiler attribute) memprof_log_dir (fts_examples.profiling.memprofiler.MemProfiler attribute) memprofilable() (fts_examples.profiling.memprofiler.MemProfiler static method) memprofile_ctx() (fts_examples.profiling.memprofiler.MemProfiler method) memprofile_fsdp_ctx() (fts_examples.profiling.memprofiler.MemProfiler method) memprofile_meta_ctx() (fts_examples.profiling.memprofiler.MemProfiler method) memprofile_snap_ctx() (fts_examples.profiling.memprofiler.MemProfiler method) MemProfiler (class in fts_examples.profiling.memprofiler) memprofiler (fts_examples.profiling.memprofiler.MemProfiler.memprofile_ctx parameter) (fts_examples.profiling.memprofiler.MemProfiler.memprofile_fsdp_ctx parameter) (fts_examples.profiling.memprofiler.MemProfiler.memprofile_meta_ctx parameter) (fts_examples.profiling.memprofiler.MemProfiler.memprofile_snap_ctx parameter)

MemProfilerCfg (class in fts_examples.profiling.config) MemProfilerFuncs (class in fts_examples.profiling.config) MemProfilerHooks (class in fts_examples.profiling.config) MemProfilerSchedule (class in fts_examples.profiling.config) ModelParallelStrategyAdapter (class in finetuning_scheduler.strategy_adapters) module finetuning_scheduler.dynamic_versioning finetuning_scheduler.fts finetuning_scheduler.fts_supporters finetuning_scheduler.strategy_adapters fts_examples.profiling.config fts_examples.profiling.memprofiler module (finetuning_scheduler.fts_supporters.ScheduleImplMixin.add_optimizer_groups parameter) (finetuning_scheduler.fts_supporters.ScheduleImplMixin.gen_ft_schedule parameter) (finetuning_scheduler.strategy_adapters.StrategyAdapter.base_ft_phase parameter) (fts_examples.profiling.memprofiler.MemProfiler attribute)

N

new_lr_scheduler (finetuning_scheduler.fts_supporters.ScheduleParsingMixin.reinit_lr_scheduler parameter)

new_optimizer (finetuning_scheduler.fts_supporters.ScheduleParsingMixin.reinit_optimizer parameter) no_decay (finetuning_scheduler.fts_supporters.ScheduleImplMixin.add_optimizer_groups parameter)

O

obj_ref (fts_examples.profiling.memprofiler.MemProfiler.connect parameter) objs (finetuning_scheduler.fts_supporters.ScheduleImplMixin.sync parameter) on_after_init_fts() (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter method) (finetuning_scheduler.strategy_adapters.StrategyAdapter method) on_before_fts_fit_start() (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter method) (finetuning_scheduler.strategy_adapters.ModelParallelStrategyAdapter method) (finetuning_scheduler.strategy_adapters.StrategyAdapter method) on_before_init_fts() (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter method) (finetuning_scheduler.strategy_adapters.ModelParallelStrategyAdapter method) (finetuning_scheduler.strategy_adapters.StrategyAdapter method) on_before_restore_optimizers_and_lrs() (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter method) (finetuning_scheduler.strategy_adapters.StrategyAdapter method) on_before_zero_grad() (finetuning_scheduler.fts.FinetuningScheduler method)

on_fit_start() (finetuning_scheduler.fts.FinetuningScheduler method) on_train_end() (finetuning_scheduler.fts.FinetuningScheduler method) on_train_epoch_start() (finetuning_scheduler.fts.FinetuningScheduler method) on_validation_end() (finetuning_scheduler.fts_supporters.FTSEarlyStopping method) optimizer (finetuning_scheduler.fts.FinetuningScheduler.on_before_zero_grad parameter) (finetuning_scheduler.fts.FinetuningScheduler.step_pg parameter) (finetuning_scheduler.fts_supporters.ScheduleImplMixin.add_optimizer_groups parameter) (finetuning_scheduler.fts_supporters.ScheduleParsingMixin.reinit_lr_scheduler parameter) (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter.optimizer_state parameter) optimizer_state() (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter method) orig_pl (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter.fts_optim_transform parameter) (finetuning_scheduler.strategy_adapters.StrategyAdapter.fts_optim_transform parameter) orig_thaw_pl (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter.fsdp_param_transform parameter)

P

param_names (finetuning_scheduler.strategy_adapters.FSDPStrategyAdapter.logical_param_translation parameter) (finetuning_scheduler.strategy_adapters.StrategyAdapter.logical_param_translation parameter) phase0_optimizer_override() (finetuning_scheduler.strategy_adapters.StrategyAdapter method) pl_module (finetuning_scheduler.fts.FinetuningScheduler.freeze_before_training parameter) (finetuning_scheduler.fts.FinetuningScheduler.on_before_zero_grad parameter) (finetuning_scheduler.fts.FinetuningScheduler.on_fit_start parameter) (finetuning_scheduler.fts.FinetuningScheduler.on_train_end parameter) (finetuning_scheduler.fts.FinetuningScheduler.on_train_epoch_start parameter) (finetuning_scheduler.fts.FinetuningScheduler.setup parameter) (finetuning_scheduler.fts_supporters.FTSEarlyStopping.on_validation_end parameter) (finetuning_scheduler.strategy_adapters.StrategyAdapter property)

pls_handle (finetuning_scheduler.strategy_adapters.StrategyAdapter property)

R

rank (fts_examples.profiling.memprofiler.MemProfiler attribute) reconnect (finetuning_scheduler.fts_supporters.CallbackResolverMixin.connect_callback parameter) reduce_transition_decisions (finetuning_scheduler.fts_supporters.FTSEarlyStopping attribute) reinit_lr_cfg (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] reinit_lr_scheduler() (finetuning_scheduler.fts_supporters.ScheduleParsingMixin method) reinit_optim_cfg (finetuning_scheduler.fts.FinetuningScheduler parameter), [1]

reinit_optimizer() (finetuning_scheduler.fts_supporters.ScheduleParsingMixin method) remove_memprofiler_hooks() (fts_examples.profiling.memprofiler.MemProfiler method) reset_mem_hooks (fts_examples.profiling.memprofiler.MemProfiler.snap parameter) restore_best (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] restore_best_ckpt() (finetuning_scheduler.fts.FinetuningScheduler method) retain_hooks_for_funcs (fts_examples.profiling.config.MemProfilerCfg parameter), [1]

S

save_dir (fts_examples.profiling.config.MemProfilerCfg parameter), [1] save_fsdp_mem_reports() (fts_examples.profiling.memprofiler.MemProfiler method) save_hook_attrs (fts_examples.profiling.config.MemProfilerCfg parameter), [1] save_on_train_epoch_end (finetuning_scheduler.fts_supporters.FTSCheckpoint attribute) save_schedule() (finetuning_scheduler.fts_supporters.ScheduleImplMixin static method) saved_tensors_funcs (fts_examples.profiling.config.MemProfilerCfg parameter), [1] (fts_examples.profiling.memprofiler.MemProfiler attribute) sched_dir (finetuning_scheduler.fts_supporters.ScheduleImplMixin.gen_implicit_schedule parameter) schedule (fts_examples.profiling.config.MemProfilerCfg parameter), [1] schedule_name (finetuning_scheduler.fts_supporters.ScheduleImplMixin.save_schedule parameter) schedule_yaml_file (finetuning_scheduler.fts_supporters.ScheduleImplMixin.load_yaml_schedule parameter) ScheduleImplMixin (class in finetuning_scheduler.fts_supporters) ScheduleParsingMixin (class in finetuning_scheduler.fts_supporters) setup() (finetuning_scheduler.fts.FinetuningScheduler method) (finetuning_scheduler.fts_supporters.FTSCheckpoint method) (finetuning_scheduler.fts_supporters.FTSEarlyStopping method)

should_transition() (finetuning_scheduler.fts.FinetuningScheduler method) snap() (fts_examples.profiling.memprofiler.MemProfiler method) snap_key (fts_examples.profiling.memprofiler.MemProfiler.cuda_allocator_history_snap parameter) stage (finetuning_scheduler.fts.FinetuningScheduler.setup parameter) state_dict (finetuning_scheduler.fts.FinetuningScheduler.load_state_dict parameter) (finetuning_scheduler.fts_supporters.FTSCheckpoint.load_state_dict parameter) state_dict() (finetuning_scheduler.fts.FinetuningScheduler method) (finetuning_scheduler.fts_supporters.FTSCheckpoint method) step() (finetuning_scheduler.fts.FinetuningScheduler method) step_pg() (finetuning_scheduler.fts.FinetuningScheduler method) strategy_adapter_cfg (finetuning_scheduler.fts.FinetuningScheduler attribute) (finetuning_scheduler.fts.FinetuningScheduler parameter), [1] StrategyAdapter (class in finetuning_scheduler.strategy_adapters) support_multiple_targets (finetuning_scheduler.fts_supporters.CallbackResolverMixin parameter), [1] sync() (finetuning_scheduler.fts_supporters.ScheduleImplMixin static method)

T

target_callback_ref (finetuning_scheduler.fts_supporters.CallbackResolverMixin parameter), [1] teardown_prof() (fts_examples.profiling.memprofiler.MemProfiler method) thaw_pl (finetuning_scheduler.strategy_adapters.StrategyAdapter.base_ft_phase parameter) thaw_to_depth() (finetuning_scheduler.fts_supporters.ScheduleImplMixin method) thawed_pl (finetuning_scheduler.fts_supporters.ScheduleImplMixin.add_optimizer_groups parameter) toggle_lightning_imports() (in module finetuning_scheduler.dynamic_versioning) track_fsdp_mem (fts_examples.profiling.config.MemProfilerCfg parameter), [1] track_inputs_target (fts_examples.profiling.memprofiler.MemProfiler.memprofile_fsdp_ctx parameter) trainer (finetuning_scheduler.fts.FinetuningScheduler.on_before_zero_grad parameter) (finetuning_scheduler.fts.FinetuningScheduler.on_fit_start parameter) (finetuning_scheduler.fts.FinetuningScheduler.on_train_end parameter) (finetuning_scheduler.fts.FinetuningScheduler.on_train_epoch_start parameter) (finetuning_scheduler.fts.FinetuningScheduler.setup parameter) (finetuning_scheduler.fts.FinetuningScheduler.should_transition parameter) (finetuning_scheduler.fts_supporters.CallbackResolverMixin.connect_callback parameter) (finetuning_scheduler.fts_supporters.FTSCheckpoint.setup parameter) (finetuning_scheduler.fts_supporters.FTSEarlyStopping.on_validation_end parameter) (finetuning_scheduler.fts_supporters.ScheduleParsingMixin.reinit_lr_scheduler parameter) (finetuning_scheduler.fts_supporters.ScheduleParsingMixin.reinit_optimizer parameter) (finetuning_scheduler.strategy_adapters.StrategyAdapter property)

U

UniqueKeyLoader (class in finetuning_scheduler.fts_supporters)

update_collect_state() (fts_examples.profiling.memprofiler.MemProfiler method) using_sharded_optimizer (finetuning_scheduler.strategy_adapters.StrategyAdapter property)