All modules for which code is available

• src.cli
• src.deploy.deploy
• src.deploy.inference
• src.deploy.quantization
• src.model.activation_function.common
• src.model.activation_function.exponential
• src.model.activation_function.factory
• src.model.activation_function.glu
• src.model.activation_function.learnable
• src.model.activation_function.rectified
• src.model.attention.common
• src.model.attention.engram
• src.model.attention.gated.deltanet_attn
• src.model.attention.gated.fox_attn
• src.model.attention.gated.gated_deltanet2_attn
• src.model.attention.gated.gated_deltanet_attn
• src.model.attention.gated.gated_softmax_attn
• src.model.attention.gated.gla_attn
• src.model.attention.gated.hgrn2_attn
• src.model.attention.gated.kda_attn
• src.model.attention.gated.retnet_attn
• src.model.attention.grouped_query_attention
• src.model.attention.latent.cca_attn
• src.model.attention.latent.gqla_attn
• src.model.attention.latent.gta_attn
• src.model.attention.latent.iha_attn
• src.model.attention.latent.mla_attn
• src.model.attention.latent.mlra_attn
• src.model.attention.latent.mtla_attn
• src.model.attention.latent.tucker_attn
• src.model.attention.ode
• src.model.attention.retnet
• src.model.attention.sigmoid
• src.model.attention.sparse.bigbird_attn
• src.model.attention.sparse.fasa_attn
• src.model.attention.sparse.longformer_attn
• src.model.attention.sparse.msa_attn
• src.model.attention.sparse.nsa_attn
• src.model.attention.sparse.sparda_attn
• src.model.attention.sparse.sparge_attn
• src.model.attention.sparse.sparse_transformer_attn
• src.model.attention.sparse.sparsek_attn
• src.model.attention.standard
• src.model.attention.titan
• src.model.embeddings.factorized_embedding
• src.model.embeddings.hope
• src.model.embeddings.rope
• src.model.norm.derf
• src.model.norm.dynamic_tanh
• src.model.norm.factory
• src.model.norm.rms
• src.model.optimizer.adamw
• src.model.optimizer.adan
• src.model.optimizer.ademamix
• src.model.optimizer.adopt
• src.model.optimizer.apollo
• src.model.optimizer.apollo_mini
• src.model.optimizer.cautious_adamw
• src.sbert.inference_sbert
• src.tokenizer.spm_spa_redpajama35
• src.training.streaming_mlm_dataset
• src.utils.device
• src.utils.gpu_temp_guard
• src.utils.storage_manager
• torch